A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Frontend] [Core] Support for sharded tensorized models (#4990)
[Frontend] [Core] Support for sharded tensorized models (#4990) Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Sanger Steel <sangersteel@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Travis Johnson committed
Jun 12, 2024 at 15:13 UTC
51602eefd38250325e541abd28f051ffd7676c3f
6 files changed
+261
-107
examples/tensorize_vllm_model.py
+60
-65
@@ -3,18 +3,12 @@ import dataclasses
3
import json
4
import os
5
import uuid
6
-from functools import partial
7
-
8
-from tensorizer import stream_io
6
7
from vllm import LLM
11
-from vllm.distributed import (init_distributed_environment,
12
- initialize_model_parallel)
8
from vllm.engine.arg_utils import EngineArgs
14
-from vllm.engine.llm_engine import LLMEngine
9
from vllm.model_executor.model_loader.tensorizer import (TensorizerArgs,
10
TensorizerConfig,
17
- serialize_vllm_model)
11
+ tensorize_vllm_model)
12
13
# yapf conflicts with isort for this docstring
14
# yapf: disable
@@ -61,6 +55,12 @@ Which downloads the model tensors from your S3 bucket and deserializes them.
55
You can also provide a `--keyfile` argument to decrypt the model weights if
56
they were serialized with encryption.
57
58
+To support distributed tensor-parallel models, each model shard will be
59
+serialized to a separate file. The tensorizer_uri is then specified as a string
60
+template with a format specifier such as '%03d' that will be rendered with the
61
+shard's rank. Sharded models serialized with this script will be named as
62
+model-rank-%03d.tensors
63
+
64
For more information on the available arguments for serializing, run
65
`python -m examples.tensorize_vllm_model serialize --help`.
66
@@ -168,77 +168,72 @@ def parse_args():
168
def deserialize():
169
llm = LLM(model=args.model,
170
load_format="tensorizer",
171
+ tensor_parallel_size=args.tensor_parallel_size,
172
model_loader_extra_config=tensorizer_config
173
)
174
return llm
175
176
177
+if __name__ == '__main__':
178
+ args = parse_args()
179
177
-args = parse_args()
178
-
179
-s3_access_key_id = (getattr(args, 's3_access_key_id', None)
180
- or os.environ.get("S3_ACCESS_KEY_ID", None))
181
-s3_secret_access_key = (getattr(args, 's3_secret_access_key', None)
182
- or os.environ.get("S3_SECRET_ACCESS_KEY", None))
183
-s3_endpoint = (getattr(args, 's3_endpoint', None)
184
- or os.environ.get("S3_ENDPOINT_URL", None))
185
-
186
-credentials = {
187
- "s3_access_key_id": s3_access_key_id,
188
- "s3_secret_access_key": s3_secret_access_key,
189
- "s3_endpoint": s3_endpoint
190
-}
180
+ s3_access_key_id = (getattr(args, 's3_access_key_id', None)
181
+ or os.environ.get("S3_ACCESS_KEY_ID", None))
182
+ s3_secret_access_key = (getattr(args, 's3_secret_access_key', None)
183
+ or os.environ.get("S3_SECRET_ACCESS_KEY", None))
184
+ s3_endpoint = (getattr(args, 's3_endpoint', None)
185
+ or os.environ.get("S3_ENDPOINT_URL", None))
186
192
-_read_stream, _write_stream = (partial(
193
- stream_io.open_stream,
194
- mode=mode,
195
- s3_access_key_id=s3_access_key_id,
196
- s3_secret_access_key=s3_secret_access_key,
197
- s3_endpoint=s3_endpoint,
198
-) for mode in ("rb", "wb+"))
187
+ credentials = {
188
+ "s3_access_key_id": s3_access_key_id,
189
+ "s3_secret_access_key": s3_secret_access_key,
190
+ "s3_endpoint": s3_endpoint
191
+ }
192
200
-model_ref = args.model
193
+ model_ref = args.model
194
202
-model_name = model_ref.split("/")[1]
195
+ model_name = model_ref.split("/")[1]
196
204
-os.environ["MASTER_ADDR"] = "127.0.0.1"
205
-os.environ["MASTER_PORT"] = "8080"
197
+ keyfile = args.keyfile if args.keyfile else None
198
207
-init_distributed_environment(world_size=1, rank=0, local_rank=0)
208
-initialize_model_parallel()
199
+ if args.model_loader_extra_config:
200
+ config = json.loads(args.model_loader_extra_config)
201
+ tensorizer_args = \
202
+ TensorizerConfig(**config)._construct_tensorizer_args()
203
+ tensorizer_args.tensorizer_uri = args.path_to_tensors
204
+ else:
205
+ tensorizer_args = None
206
210
-keyfile = args.keyfile if args.keyfile else None
207
+ if args.command == "serialize":
208
+ eng_args_dict = {f.name: getattr(args, f.name) for f in
209
+ dataclasses.fields(EngineArgs)}
210
211
+ engine_args = EngineArgs.from_cli_args(
212
+ argparse.Namespace(**eng_args_dict)
213
+ )
214
213
-if args.model_loader_extra_config:
214
- config = json.loads(args.model_loader_extra_config)
215
- tensorizer_args = TensorizerConfig(**config)._construct_tensorizer_args()
216
- tensorizer_args.tensorizer_uri = args.path_to_tensors
217
-else:
218
- tensorizer_args = None
219
-
220
-if args.command == "serialize":
221
- eng_args_dict = {f.name: getattr(args, f.name) for f in
222
- dataclasses.fields(EngineArgs)}
223
-
224
- engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
225
- engine = LLMEngine.from_engine_args(engine_args)
215
+ input_dir = args.serialized_directory.rstrip('/')
216
+ suffix = args.suffix if args.suffix else uuid.uuid4().hex
217
+ base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
218
+ if engine_args.tensor_parallel_size > 1:
219
+ model_path = f"{base_path}/model-rank-%03d.tensors"
220
+ else:
221
+ model_path = f"{base_path}/model.tensors"
222
227
- input_dir = args.serialized_directory.rstrip('/')
228
- suffix = args.suffix if args.suffix else uuid.uuid4().hex
229
- base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
230
- model_path = f"{base_path}/model.tensors"
231
- tensorizer_config = TensorizerConfig(
232
- tensorizer_uri=model_path,
233
- **credentials)
234
- serialize_vllm_model(engine, tensorizer_config, keyfile)
235
-elif args.command == "deserialize":
236
- if not tensorizer_args:
223
tensorizer_config = TensorizerConfig(
238
- tensorizer_uri=args.path_to_tensors,
239
- encryption_keyfile = keyfile,
240
- **credentials
241
- )
242
- deserialize()
243
-else:
244
- raise ValueError("Either serialize or deserialize must be specified.")
224
+ tensorizer_uri=model_path,
225
+ encryption_keyfile=keyfile,
226
+ **credentials)
227
+
228
+ tensorize_vllm_model(engine_args, tensorizer_config)
229
+
230
+ elif args.command == "deserialize":
231
+ if not tensorizer_args:
232
+ tensorizer_config = TensorizerConfig(
233
+ tensorizer_uri=args.path_to_tensors,
234
+ encryption_keyfile = keyfile,
235
+ **credentials
236
+ )
237
+ deserialize()
238
+ else:
239
+ raise ValueError("Either serialize or deserialize must be specified.")
tests/tensorizer_loader/test_tensorizer.py
+90
-9
@@ -1,21 +1,27 @@
1
import json
2
import os
3
+import pathlib
4
import subprocess
5
from unittest.mock import MagicMock, patch
6
7
import openai
8
import pytest
9
import ray
10
+import torch
11
+from tensorizer import EncryptionParams
12
13
from vllm import SamplingParams
14
+from vllm.engine.arg_utils import EngineArgs
15
# yapf: disable
16
from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
17
TensorSerializer,
18
is_vllm_tensorized,
19
load_with_tensorizer,
20
open_stream,
17
- serialize_vllm_model)
21
+ serialize_vllm_model,
22
+ tensorize_vllm_model)
23
24
+from ..conftest import VllmRunner, cleanup
25
from ..utils import ServerRunner
26
27
# yapf conflicts with isort for this docstring
@@ -42,6 +48,20 @@ def is_curl_installed():
48
except (subprocess.CalledProcessError, FileNotFoundError):
49
return False
50
51
+def get_torch_model(vllm_runner: VllmRunner):
52
+ return vllm_runner \
53
+ .model \
54
+ .llm_engine \
55
+ .model_executor \
56
+ .driver_worker \
57
+ .model_runner \
58
+ .model
59
+
60
+def write_keyfile(keyfile_path: str):
61
+ encryption_params = EncryptionParams.random()
62
+ pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
63
+ with open(keyfile_path, 'wb') as f:
64
+ f.write(encryption_params.key)
65
66
@pytest.fixture(autouse=True)
67
def tensorizer_config():
@@ -88,12 +108,17 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
108
with vllm_runner(model_ref) as vllm_model:
109
model_path = tmp_path / (model_ref + ".tensors")
110
key_path = tmp_path / (model_ref + ".key")
111
+ write_keyfile(key_path)
112
+
113
outputs = vllm_model.generate(prompts, sampling_params)
114
93
- config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
94
- serialize_vllm_model(vllm_model.model.llm_engine,
95
- config_for_serializing,
96
- encryption_key_path=key_path)
115
+ config_for_serializing = TensorizerConfig(
116
+ tensorizer_uri=model_path,
117
+ encryption_keyfile=key_path
118
+ )
119
+ serialize_vllm_model(get_torch_model(vllm_model),
120
+ config_for_serializing)
121
+
122
123
config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
124
encryption_keyfile=key_path)
@@ -145,7 +170,7 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
170
with vllm_runner(model_ref, ) as vllm_model:
171
model_path = tmp_path / (model_ref + ".tensors")
172
148
- serialize_vllm_model(vllm_model.model.llm_engine,
173
+ serialize_vllm_model(get_torch_model(vllm_model),
174
TensorizerConfig(tensorizer_uri=model_path))
175
176
with vllm_runner(
@@ -180,7 +205,7 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
205
with vllm_runner(model_ref, ) as vllm_model:
206
model_path = tmp_path / (model_ref + ".tensors")
207
183
- serialize_vllm_model(vllm_model.model.llm_engine,
208
+ serialize_vllm_model(get_torch_model(vllm_model),
209
TensorizerConfig(tensorizer_uri=model_path))
210
211
model_loader_extra_config = {
@@ -224,7 +249,9 @@ def test_raise_value_error_on_invalid_load_format(vllm_runner):
249
model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"))
250
251
227
-def test_tensorizer_with_tp(vllm_runner):
252
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
253
+ reason="Requires 2 GPUs")
254
+def test_tensorizer_with_tp_path_without_template(vllm_runner):
255
with pytest.raises(ValueError):
256
model_ref = "EleutherAI/pythia-1.4b"
257
tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
@@ -238,8 +265,62 @@ def test_tensorizer_with_tp(vllm_runner):
265
s3_endpoint="object.ord1.coreweave.com",
266
),
267
tensor_parallel_size=2,
268
+ disable_custom_all_reduce=True,
269
)
270
271
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
272
+ reason="Requires 2 GPUs")
273
+def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
274
+ tmp_path):
275
+ model_ref = "EleutherAI/pythia-1.4b"
276
+ # record outputs from un-sharded un-tensorized model
277
+ base_model = vllm_runner(
278
+ model_ref,
279
+ disable_custom_all_reduce=True,
280
+ enforce_eager=True,
281
+ )
282
+ outputs = base_model.generate(prompts, sampling_params)
283
+
284
+ base_model.model.llm_engine.model_executor.shutdown()
285
+ del base_model
286
+ cleanup()
287
+ ray.shutdown()
288
+
289
+ # load model with two shards and serialize with encryption
290
+ model_path = str(tmp_path / (model_ref + "-%02d.tensors"))
291
+ key_path = tmp_path / (model_ref + ".key")
292
+
293
+ tensorizer_config = TensorizerConfig(
294
+ tensorizer_uri=model_path,
295
+ encryption_keyfile=key_path,
296
+ )
297
+
298
+ tensorize_vllm_model(
299
+ engine_args=EngineArgs(
300
+ model=model_ref,
301
+ tensor_parallel_size=2,
302
+ disable_custom_all_reduce=True,
303
+ enforce_eager=True,
304
+ ),
305
+ tensorizer_config=tensorizer_config,
306
+ )
307
+ assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
308
+ assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
309
+ cleanup()
310
+ ray.shutdown()
311
+
312
+ loaded_vllm_model = vllm_runner(
313
+ model_ref,
314
+ tensor_parallel_size=2,
315
+ load_format="tensorizer",
316
+ disable_custom_all_reduce=True,
317
+ enforce_eager=True,
318
+ model_loader_extra_config=tensorizer_config)
319
+
320
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
321
+
322
+ assert outputs == deserialized_outputs
323
+
324
325
def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
326
model_ref = "facebook/opt-125m"
@@ -248,7 +329,7 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
329
330
with vllm_runner(model_ref) as vllm_model:
331
outputs = vllm_model.generate(prompts, sampling_params)
251
- serialize_vllm_model(vllm_model.model.llm_engine, config)
332
+ serialize_vllm_model(get_torch_model(vllm_model), config)
333
334
assert is_vllm_tensorized(config)
335
vllm/model_executor/model_loader/loader.py
+17
-1
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.quantization.base_config import (
24
QuantizationConfig)
25
from vllm.model_executor.model_loader.tensorizer import (
26
TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
27
- tensorizer_weights_iterator)
27
+ serialize_vllm_model, tensorizer_weights_iterator)
28
from vllm.model_executor.model_loader.utils import (get_model_architecture,
29
set_default_torch_dtype)
30
from vllm.model_executor.model_loader.weight_utils import (
@@ -392,6 +392,12 @@ class TensorizerLoader(BaseModelLoader):
392
cache_config: CacheConfig) -> nn.Module:
393
self._verify_config(model_config, parallel_config)
394
395
+ if parallel_config.tensor_parallel_size > 1:
396
+ from vllm.distributed import get_tensor_model_parallel_rank
397
+ self.tensorizer_config.tensorizer_uri = \
398
+ self.tensorizer_config.tensorizer_uri \
399
+ % get_tensor_model_parallel_rank()
400
+
401
if is_vllm_tensorized(self.tensorizer_config):
402
return self._load_model_serialized(model_config, device_config,
403
lora_config,
@@ -402,6 +408,16 @@ class TensorizerLoader(BaseModelLoader):
408
vision_language_config,
409
cache_config)
410
411
+ @staticmethod
412
+ def save_model(
413
+ model: torch.nn.Module,
414
+ tensorizer_config: TensorizerConfig,
415
+ ) -> None:
416
+ serialize_vllm_model(
417
+ model=model,
418
+ tensorizer_config=tensorizer_config,
419
+ )
420
+
421
422
class ShardedStateLoader(BaseModelLoader):
423
"""
vllm/model_executor/model_loader/tensorizer.py
+75
-32
@@ -2,11 +2,11 @@ import argparse
2
import dataclasses
3
import io
4
import os
5
+import re
6
import time
6
-import typing
7
from dataclasses import dataclass
8
from functools import partial
9
-from typing import Generator, Optional, Tuple, Type, Union
9
+from typing import BinaryIO, Generator, Optional, Tuple, Type, Union
10
11
import torch
12
from torch import nn
@@ -14,6 +14,7 @@ from transformers import PretrainedConfig
14
15
import vllm.envs as envs
16
from vllm.config import ModelConfig, ParallelConfig
17
+from vllm.engine.arg_utils import EngineArgs
18
from vllm.engine.llm_engine import LLMEngine
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.quantization.base_config import (
@@ -48,8 +49,7 @@ logger = init_logger(__name__)
49
50
@dataclass
51
class TensorizerConfig:
51
- tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
52
- str, bytes, os.PathLike, int]
52
+ tensorizer_uri: str
53
vllm_tensorized: Optional[bool] = False
54
verify_hash: Optional[bool] = False
55
num_readers: Optional[int] = None
@@ -60,6 +60,12 @@ class TensorizerConfig:
60
model_class: Optional[Type[torch.nn.Module]] = None
61
hf_config: Optional[PretrainedConfig] = None
62
dtype: Optional[Union[str, torch.dtype]] = None
63
+ _is_sharded: bool = False
64
+
65
+ def __post_init__(self):
66
+ # check if the configuration is for a sharded vLLM model
67
+ self._is_sharded = isinstance(self.tensorizer_uri, str) \
68
+ and re.search(r'%0\dd', self.tensorizer_uri) is not None
69
70
def _construct_tensorizer_args(self) -> "TensorizerArgs":
71
tensorizer_args = {
@@ -78,13 +84,12 @@ class TensorizerConfig:
84
self,
85
parallel_config: "ParallelConfig",
86
) -> None:
81
- if (parallel_config.tensor_parallel_size > 1
82
- and self.tensorizer_uri is not None):
87
+ if parallel_config.tensor_parallel_size > 1 \
88
+ and not self._is_sharded:
89
raise ValueError(
84
- "Loading to multiple GPUs is not currently supported with "
85
- "vLLM-serialized models. Please set tensor_parallel_size=1."
86
- " or use a non-vLLM-serialized model, such as a "
87
- "serialized Hugging Face `PretrainedModel`.")
90
+ "For a sharded model, tensorizer_uri should include a"
91
+ " string format template like '%04d' to be formatted"
92
+ " with the rank of the shard")
93
94
def verify_with_model_config(self, model_config: "ModelConfig") -> None:
95
if (model_config.quantization is not None
@@ -102,8 +107,8 @@ def load_with_tensorizer(tensorizer_config: TensorizerConfig,
107
108
@dataclass
109
class TensorizerArgs:
105
- tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
106
- str, bytes, os.PathLike, int]
110
+ tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, BinaryIO, str,
111
+ bytes, os.PathLike, int]
112
vllm_tensorized: Optional[bool] = False
113
verify_hash: Optional[bool] = False
114
num_readers: Optional[int] = None
@@ -332,6 +337,7 @@ class TensorizerAgent:
337
) as stream, TensorDeserializer(
338
stream,
339
dtype=self.tensorizer_config.dtype,
340
+ device=f'cuda:{torch.cuda.current_device()}',
341
**self.tensorizer_args.deserializer_params) as deserializer:
342
deserializer.load_into_module(self.model)
343
end = time.perf_counter()
@@ -400,33 +406,70 @@ def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
406
return False
407
408
403
-def get_pretensorized_vllm_model(engine: "LLMEngine") -> nn.Module:
404
- model = (engine.model_executor.driver_worker.model_runner.model)
409
+def serialize_vllm_model(
410
+ model: nn.Module,
411
+ tensorizer_config: TensorizerConfig,
412
+) -> nn.Module:
413
model.register_parameter(
414
"vllm_tensorized_marker",
415
nn.Parameter(torch.tensor((1, ), device="meta"), requires_grad=False))
408
- return model
409
-
410
-
411
-def serialize_vllm_model(engine: "LLMEngine",
412
- tensorizer_config : TensorizerConfig,
413
- encryption_key_path: Optional[str] = None) \
414
- -> nn.Module:
415
-
416
- model = get_pretensorized_vllm_model(engine)
416
tensorizer_args = tensorizer_config._construct_tensorizer_args()
417
+
418
encryption_params = None
419
- if encryption_key_path is not None:
420
- encryption_params = EncryptionParams.random()
421
- with _write_stream(encryption_key_path,
422
- **tensorizer_args.stream_params) as stream:
423
- stream.write(encryption_params.key)
419
+ if (keyfile := tensorizer_config.encryption_keyfile) is not None:
420
+ with open(keyfile, "rb") as f:
421
+ key = f.read()
422
+ encryption_params = EncryptionParams(key=key)
423
425
- with _write_stream(tensorizer_args.tensorizer_uri,
426
- **tensorizer_args.stream_params) as stream:
424
+ output_file = tensorizer_args.tensorizer_uri
425
+ if tensorizer_config._is_sharded:
426
+ from vllm.distributed import get_tensor_model_parallel_rank
427
+ output_file = output_file % get_tensor_model_parallel_rank()
428
+
429
+ with _write_stream(output_file, **tensorizer_args.stream_params) as stream:
430
serializer = TensorSerializer(stream, encryption=encryption_params)
431
serializer.write_module(model)
432
serializer.close()
430
- logger.info("Successfully serialized model to %s",
431
- str(tensorizer_args.tensorizer_uri))
433
+ logger.info("Successfully serialized model to %s", str(output_file))
434
return model
435
+
436
+
437
+def tensorize_vllm_model(engine_args: EngineArgs,
438
+ tensorizer_config: TensorizerConfig,
439
+ generate_keyfile: bool = True):
440
+ """Utility to load a model and then serialize it with Tensorizer
441
+
442
+ Intended to be used separately from running a vLLM server since it
443
+ creates its own Engine instance.
444
+ """
445
+ engine_config = engine_args.create_engine_config()
446
+ tensorizer_config.verify_with_model_config(engine_config.model_config)
447
+ tensorizer_config.verify_with_parallel_config(
448
+ engine_config.parallel_config)
449
+
450
+ # generate the encryption key before creating the engine to support sharding
451
+ if generate_keyfile and (keyfile :=
452
+ tensorizer_config.encryption_keyfile) is not None:
453
+ encryption_params = EncryptionParams.random()
454
+ with _write_stream(
455
+ keyfile,
456
+ s3_access_key_id=tensorizer_config.s3_access_key_id,
457
+ s3_secret_access_key=tensorizer_config.s3_secret_access_key,
458
+ s3_endpoint=tensorizer_config.s3_endpoint,
459
+ ) as stream:
460
+ stream.write(encryption_params.key)
461
+
462
+ engine = LLMEngine.from_engine_args(engine_args)
463
+ if tensorizer_config._is_sharded:
464
+ # if the engine is a distributed engine (for tensor parallel) then each
465
+ # worker shard needs to serialize its part of the model.
466
+ engine.model_executor._run_workers(
467
+ "save_tensorized_model",
468
+ tensorizer_config=tensorizer_config,
469
+ )
470
+ else:
471
+ # with a single worker, we can get to the underlying model directly
472
+ serialize_vllm_model(
473
+ engine.model_executor.driver_worker.model_runner.model,
474
+ tensorizer_config,
475
+ )
vllm/worker/model_runner.py
+11
@@ -20,6 +20,7 @@ from vllm.lora.request import LoRARequest
20
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
21
from vllm.model_executor import SamplingMetadata
22
from vllm.model_executor.model_loader import get_model
23
+from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.sampling_params import SamplingParams
26
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
@@ -222,6 +223,16 @@ class ModelRunner:
223
max_size=max_size,
224
)
225
226
+ def save_tensorized_model(
227
+ self,
228
+ tensorizer_config: TensorizerConfig,
229
+ ) -> None:
230
+ from vllm.model_executor.model_loader.loader import TensorizerLoader
231
+ TensorizerLoader.save_model(
232
+ self.model,
233
+ tensorizer_config=tensorizer_config,
234
+ )
235
+
236
def get_max_block_per_batch(self) -> int:
237
block_size = self.block_size
238
return (self.max_seq_len_to_capture + block_size - 1) // block_size
vllm/worker/worker.py
+8
@@ -15,6 +15,7 @@ from vllm.distributed import (broadcast_tensor_dict,
15
set_custom_all_reduce)
16
from vllm.lora.request import LoRARequest
17
from vllm.model_executor import set_random_seed
18
+from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19
from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
20
from vllm.worker.cache_engine import CacheEngine
21
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
@@ -132,6 +133,13 @@ class Worker(WorkerBase):
133
max_size=max_size,
134
)
135
136
+ def save_tensorized_model(
137
+ self,
138
+ tensorizer_config: TensorizerConfig,
139
+ ) -> None:
140
+ self.model_runner.save_tensorized_model(
141
+ tensorizer_config=tensorizer_config, )
142
+
143
@torch.inference_mode()
144
def determine_num_available_blocks(self) -> Tuple[int, int]:
145
"""Profiles the peak memory usage of the model to determine how many