A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Doc] Add better clarity for tensorizer usage (#4090)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Sanger Steel committed
Apr 15, 2024 at 16:28 UTC
d619ae2d19c41d9aa8f68fa0e5e32cc410dc2522
3 files changed
+46
-22
docs/source/models/engine_args.rst
+1
-1
@@ -45,7 +45,7 @@ Below, you can find an explanation of every engine argument for vLLM:
45
* "safetensors" will load the weights in the safetensors format.
46
* "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading.
47
* "dummy" will initialize the weights with random values, mainly for profiling.
48
- * "tensorizer" will load serialized weights using `CoreWeave's Tensorizer model deserializer. <https://github.com/coreweave/tensorizer>`_. See `tensorized_vllm_model.py` in the examples folder to serialize a vLLM model, and for more information. Tensorizer support for vLLM can be installed with `pip install vllm[tensorizer]`.
48
+ * "tensorizer" will load serialized weights using `CoreWeave's Tensorizer model deserializer. <https://github.com/coreweave/tensorizer>`_ See `examples/tensorize_vllm_model.py <https://github.com/vllm-project/vllm/blob/main/examples/tensorize_vllm_model.py>`_ to serialize a vLLM model, and for more information.
49
50
.. option:: --dtype {auto,half,float16,bfloat16,float,float32}
51
examples/tensorize_vllm_model.py
+44
-16
@@ -23,14 +23,16 @@ from vllm.model_executor.tensorizer_loader import TensorizerArgs
23
# yapf: disable
24
"""
25
tensorize_vllm_model.py is a script that can be used to serialize and
26
-deserialize vLLM models. These models can be loaded using tensorizer directly
27
-to the GPU extremely quickly. Tensor encryption and decryption is also
28
-supported, although libsodium must be installed to use it. Install
29
-vllm with tensorizer support using `pip install vllm[tensorizer]`.
26
+deserialize vLLM models. These models can be loaded using tensorizer
27
+to the GPU extremely quickly over an HTTP/HTTPS endpoint, an S3 endpoint,
28
+or locally. Tensor encryption and decryption is also supported, although
29
+libsodium must be installed to use it. Install vllm with tensorizer support
30
+using `pip install vllm[tensorizer]`.
31
31
-To serialize a model, you can run something like this:
32
+To serialize a model, install vLLM from source, then run something
33
+like this from the root level of this repository:
34
33
-python tensorize_vllm_model.py \
35
+python -m examples.tensorize_vllm_model \
36
--model EleutherAI/gpt-j-6B \
37
--dtype float16 \
38
serialize \
@@ -38,31 +40,57 @@ python tensorize_vllm_model.py \
40
--suffix vllm
41
42
Which downloads the model from HuggingFace, loads it into vLLM, serializes it,
41
-and saves it to your S3 bucket. A local directory can also be used.
43
+and saves it to your S3 bucket. A local directory can also be used. This
44
+assumes your S3 credentials are specified as environment variables
45
+in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
46
+To provide S3 credentials directly, you can provide `--s3-access-key-id` and
47
+`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this
48
+script.
49
50
You can also encrypt the model weights with a randomly-generated key by
51
providing a `--keyfile` argument.
52
46
-To deserialize a model, you can run something like this:
53
+To deserialize a model, you can run something like this from the root
54
+level of this repository:
55
48
-python tensorize_vllm_model.py \
56
+python -m examples.tensorize_vllm_model \
57
--model EleutherAI/gpt-j-6B \
58
--dtype float16 \
59
deserialize \
60
--path-to-tensors s3://my-bucket/vllm/EleutherAI/gpt-j-6B/vllm/model.tensors
61
62
Which downloads the model tensors from your S3 bucket and deserializes them.
55
-To provide S3 credentials, you can provide `--s3-access-key-id` and
56
-`--s3-secret-access-key`, as well as `--s3-endpoint` as CLI args to this script,
57
-the OpenAI entrypoint, as arguments for LLM(), or as environment variables
58
-in the form of `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`, and `S3_ENDPOINT`.
59
-
63
64
You can also provide a `--keyfile` argument to decrypt the model weights if
65
they were serialized with encryption.
66
64
-For more information on the available arguments, run
65
-`python tensorize_vllm_model.py --help`.
67
+For more information on the available arguments for serializing, run
68
+`python -m examples.tensorize_vllm_model serialize --help`.
69
+
70
+Or for deserializing:
71
+
72
+`python -m examples.tensorize_vllm_model deserialize --help`.
73
+
74
+Once a model is serialized, it can be used to load the model when running the
75
+OpenAI inference client at `vllm/entrypoints/openai/api_server.py` by providing
76
+the `--tensorizer-uri` CLI argument that is functionally the same as the
77
+`--path-to-tensors` argument in this script, along with `--vllm-tensorized`, to
78
+signify that the model to be deserialized is a vLLM model, rather than a
79
+HuggingFace `PreTrainedModel`, which can also be deserialized using tensorizer
80
+in the same inference server, albeit without the speed optimizations. To
81
+deserialize an encrypted file, the `--encryption-keyfile` argument can be used
82
+to provide the path to the keyfile used to encrypt the model weights. For
83
+information on all the arguments that can be used to configure tensorizer's
84
+deserialization, check out the tensorizer options argument group in the
85
+`vllm/entrypoints/openai/api_server.py` script with `--help`.
86
+
87
+Tensorizer can also be invoked with the `LLM` class directly to load models:
88
+
89
+ llm = LLM(model="facebook/opt-125m",
90
+ load_format="tensorizer",
91
+ tensorizer_uri=path_to_opt_tensors,
92
+ num_readers=3,
93
+ vllm_tensorized=True)
94
"""
95
96
vllm/model_executor/tensorizer_loader.py
+1
-5
@@ -126,7 +126,6 @@ class TensorizerArgs:
126
"s3_endpoint": self.s3_endpoint,
127
}
128
129
- # Omitting self.dtype and self.device as this behaves weirdly
129
self.deserializer_params = {
130
"verify_hash": self.verify_hash,
131
"encryption": self.encryption_keyfile,
@@ -145,7 +144,7 @@ class TensorizerArgs:
144
parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
145
"""Tensorizer CLI arguments"""
146
148
- # Create the argument group
147
+ # Tensorizer options arg group
148
group = parser.add_argument_group(
149
'tensorizer options',
150
description=('Options for configuring the behavior of the'
@@ -205,9 +204,7 @@ class TensorizerArgs:
204
205
@classmethod
206
def from_cli_args(cls, args: argparse.Namespace) -> "TensorizerArgs":
208
- # Get the list of attributes of this dataclass.
207
attrs = [attr.name for attr in dataclasses.fields(cls)]
210
- # Set the attributes from the parsed arguments.
208
tensorizer_args = cls(**{
209
attr: getattr(args, attr)
210
for attr in attrs if hasattr(args, attr)
@@ -291,7 +288,6 @@ class TensorizerAgent:
288
nn.Module: The deserialized model.
289
"""
290
before_mem = get_mem_usage()
294
- # Lazy load the tensors from S3 into the model.
291
start = time.perf_counter()
292
with open_stream(
293
self.tensorizer_args.tensorizer_uri,