@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 715c2d854d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] [Core] Tensorizer: support dynamic `num_readers`, update version (#4467)

Alpay Ariyak committed Apr 30, 2024 at 19:32 UTC 715c2d854d56f2026c31f126a90e6e7859434a50
3 files changed +12 -9
requirements-dev.txt
+1 -1
@@ -14,7 +14,7 @@ types-setuptools
14
15 # testing
16 pytest
17 -tensorizer==2.9.0a0
17 +tensorizer==2.9.0
18 pytest-forked
19 pytest-asyncio
20 pytest-rerunfailures
setup.py
+1 -1
@@ -408,7 +408,7 @@ setup(
408 install_requires=get_requirements(),
409 ext_modules=ext_modules,
410 extras_require={
411 - "tensorizer": ["tensorizer==2.9.0a1"],
411 + "tensorizer": ["tensorizer==2.9.0"],
412 },
413 cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
414 package_data=package_data,
vllm/model_executor/model_loader/tensorizer.py
+10 -7
@@ -44,7 +44,7 @@ class TensorizerConfig:
44 str, bytes, os.PathLike, int]
45 vllm_tensorized: bool
46 verify_hash: Optional[bool] = False
47 - num_readers: Optional[int] = 1
47 + num_readers: Optional[int] = None
48 encryption_keyfile: Optional[str] = None
49 s3_access_key_id: Optional[str] = None
50 s3_secret_access_key: Optional[str] = None
@@ -104,7 +104,7 @@ class TensorizerArgs:
104 str, bytes, os.PathLike, int]
105 vllm_tensorized: bool
106 verify_hash: Optional[bool] = False
107 - num_readers: Optional[int] = 1
107 + num_readers: Optional[int] = None
108 encryption_keyfile: Optional[str] = None
109 s3_access_key_id: Optional[str] = None
110 s3_secret_access_key: Optional[str] = None
@@ -125,8 +125,9 @@ class TensorizerArgs:
125 the hashes stored in the metadata. A `HashMismatchError` will be
126 raised if any of the hashes do not match.
127 num_readers: Controls how many threads are allowed to read concurrently
128 - from the source file. Default is 1. This greatly increases
129 - performance.
128 + from the source file. Default is `None`, which will dynamically set
129 + the number of readers based on the number of available
130 + resources and model size. This greatly increases performance.
131 encryption_keyfile: File path to a binary file containing a
132 binary key to use for decryption. `None` (the default) means
133 no decryption. See the example script in
@@ -199,10 +200,12 @@ class TensorizerArgs:
200 "use for decryption. Can be a file path or S3 network URI.")
201 group.add_argument(
202 "--num-readers",
202 - default=1,
203 + default=None,
204 type=int,
205 help="Controls how many threads are allowed to read concurrently "
205 - "from the source file.")
206 + "from the source file. Default is `None`, which will dynamically "
207 + "set the number of readers based on the available resources "
208 + "and model size. This greatly increases performance.")
209 group.add_argument(
210 "--s3-access-key-id",
211 default=None,
@@ -337,7 +340,7 @@ class TensorizerAgent:
340 per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
341 after_mem = get_mem_usage()
342 deserializer.close()
340 - logger.info("Deserialized %s in %0.2fs, %f/s", total_bytes_str,
343 + logger.info("Deserialized %s in %0.2fs, %s/s", total_bytes_str,
344 end - start, per_second)
345 logger.info("Memory usage before: %s", before_mem)
346 logger.info("Memory usage after: %s", after_mem)