A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] [Core] Tensorizer: support dynamic `num_readers`, update version (#4467)
Alpay Ariyak committed
Apr 30, 2024 at 19:32 UTC
715c2d854d56f2026c31f126a90e6e7859434a50
3 files changed
+12
-9
requirements-dev.txt
+1
-1
@@ -14,7 +14,7 @@ types-setuptools
14
15
# testing
16
pytest
17
-tensorizer==2.9.0a0
17
+tensorizer==2.9.0
18
pytest-forked
19
pytest-asyncio
20
pytest-rerunfailures
setup.py
+1
-1
@@ -408,7 +408,7 @@ setup(
408
install_requires=get_requirements(),
409
ext_modules=ext_modules,
410
extras_require={
411
- "tensorizer": ["tensorizer==2.9.0a1"],
411
+ "tensorizer": ["tensorizer==2.9.0"],
412
},
413
cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
414
package_data=package_data,
vllm/model_executor/model_loader/tensorizer.py
+10
-7
@@ -44,7 +44,7 @@ class TensorizerConfig:
44
str, bytes, os.PathLike, int]
45
vllm_tensorized: bool
46
verify_hash: Optional[bool] = False
47
- num_readers: Optional[int] = 1
47
+ num_readers: Optional[int] = None
48
encryption_keyfile: Optional[str] = None
49
s3_access_key_id: Optional[str] = None
50
s3_secret_access_key: Optional[str] = None
@@ -104,7 +104,7 @@ class TensorizerArgs:
104
str, bytes, os.PathLike, int]
105
vllm_tensorized: bool
106
verify_hash: Optional[bool] = False
107
- num_readers: Optional[int] = 1
107
+ num_readers: Optional[int] = None
108
encryption_keyfile: Optional[str] = None
109
s3_access_key_id: Optional[str] = None
110
s3_secret_access_key: Optional[str] = None
@@ -125,8 +125,9 @@ class TensorizerArgs:
125
the hashes stored in the metadata. A `HashMismatchError` will be
126
raised if any of the hashes do not match.
127
num_readers: Controls how many threads are allowed to read concurrently
128
- from the source file. Default is 1. This greatly increases
129
- performance.
128
+ from the source file. Default is `None`, which will dynamically set
129
+ the number of readers based on the number of available
130
+ resources and model size. This greatly increases performance.
131
encryption_keyfile: File path to a binary file containing a
132
binary key to use for decryption. `None` (the default) means
133
no decryption. See the example script in
@@ -199,10 +200,12 @@ class TensorizerArgs:
200
"use for decryption. Can be a file path or S3 network URI.")
201
group.add_argument(
202
"--num-readers",
202
- default=1,
203
+ default=None,
204
type=int,
205
help="Controls how many threads are allowed to read concurrently "
205
- "from the source file.")
206
+ "from the source file. Default is `None`, which will dynamically "
207
+ "set the number of readers based on the available resources "
208
+ "and model size. This greatly increases performance.")
209
group.add_argument(
210
"--s3-access-key-id",
211
default=None,
@@ -337,7 +340,7 @@ class TensorizerAgent:
340
per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
341
after_mem = get_mem_usage()
342
deserializer.close()
340
- logger.info("Deserialized %s in %0.2fs, %f/s", total_bytes_str,
343
+ logger.info("Deserialized %s in %0.2fs, %s/s", total_bytes_str,
344
end - start, per_second)
345
logger.info("Memory usage before: %s", before_mem)
346
logger.info("Memory usage after: %s", after_mem)