@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51602eefd3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Frontend] [Core] Support for sharded tensorized models (#4990)

[Frontend] [Core] Support for sharded tensorized models (#4990) Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Sanger Steel <sangersteel@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Travis Johnson committed Jun 12, 2024 at 15:13 UTC 51602eefd38250325e541abd28f051ffd7676c3f
6 files changed +261 -107
examples/tensorize_vllm_model.py
+60 -65
@@ -3,18 +3,12 @@ import dataclasses
3 import json
4 import os
5 import uuid
6 -from functools import partial
7 -
8 -from tensorizer import stream_io
6
7 from vllm import LLM
11 -from vllm.distributed import (init_distributed_environment,
12 - initialize_model_parallel)
8 from vllm.engine.arg_utils import EngineArgs
14 -from vllm.engine.llm_engine import LLMEngine
9 from vllm.model_executor.model_loader.tensorizer import (TensorizerArgs,
10 TensorizerConfig,
17 - serialize_vllm_model)
11 + tensorize_vllm_model)
12
13 # yapf conflicts with isort for this docstring
14 # yapf: disable
@@ -61,6 +55,12 @@ Which downloads the model tensors from your S3 bucket and deserializes them.
55 You can also provide a `--keyfile` argument to decrypt the model weights if
56 they were serialized with encryption.
57
58 +To support distributed tensor-parallel models, each model shard will be
59 +serialized to a separate file. The tensorizer_uri is then specified as a string
60 +template with a format specifier such as '%03d' that will be rendered with the
61 +shard's rank. Sharded models serialized with this script will be named as
62 +model-rank-%03d.tensors
63 +
64 For more information on the available arguments for serializing, run
65 `python -m examples.tensorize_vllm_model serialize --help`.
66
@@ -168,77 +168,72 @@ def parse_args():
168 def deserialize():
169 llm = LLM(model=args.model,
170 load_format="tensorizer",
171 + tensor_parallel_size=args.tensor_parallel_size,
172 model_loader_extra_config=tensorizer_config
173 )
174 return llm
175
176
177 +if __name__ == '__main__':
178 + args = parse_args()
179
177 -args = parse_args()
178 -
179 -s3_access_key_id = (getattr(args, 's3_access_key_id', None)
180 - or os.environ.get("S3_ACCESS_KEY_ID", None))
181 -s3_secret_access_key = (getattr(args, 's3_secret_access_key', None)
182 - or os.environ.get("S3_SECRET_ACCESS_KEY", None))
183 -s3_endpoint = (getattr(args, 's3_endpoint', None)
184 - or os.environ.get("S3_ENDPOINT_URL", None))
185 -
186 -credentials = {
187 - "s3_access_key_id": s3_access_key_id,
188 - "s3_secret_access_key": s3_secret_access_key,
189 - "s3_endpoint": s3_endpoint
190 -}
180 + s3_access_key_id = (getattr(args, 's3_access_key_id', None)
181 + or os.environ.get("S3_ACCESS_KEY_ID", None))
182 + s3_secret_access_key = (getattr(args, 's3_secret_access_key', None)
183 + or os.environ.get("S3_SECRET_ACCESS_KEY", None))
184 + s3_endpoint = (getattr(args, 's3_endpoint', None)
185 + or os.environ.get("S3_ENDPOINT_URL", None))
186
192 -_read_stream, _write_stream = (partial(
193 - stream_io.open_stream,
194 - mode=mode,
195 - s3_access_key_id=s3_access_key_id,
196 - s3_secret_access_key=s3_secret_access_key,
197 - s3_endpoint=s3_endpoint,
198 -) for mode in ("rb", "wb+"))
187 + credentials = {
188 + "s3_access_key_id": s3_access_key_id,
189 + "s3_secret_access_key": s3_secret_access_key,
190 + "s3_endpoint": s3_endpoint
191 + }
192
200 -model_ref = args.model
193 + model_ref = args.model
194
202 -model_name = model_ref.split("/")[1]
195 + model_name = model_ref.split("/")[1]
196
204 -os.environ["MASTER_ADDR"] = "127.0.0.1"
205 -os.environ["MASTER_PORT"] = "8080"
197 + keyfile = args.keyfile if args.keyfile else None
198
207 -init_distributed_environment(world_size=1, rank=0, local_rank=0)
208 -initialize_model_parallel()
199 + if args.model_loader_extra_config:
200 + config = json.loads(args.model_loader_extra_config)
201 + tensorizer_args = \
202 + TensorizerConfig(**config)._construct_tensorizer_args()
203 + tensorizer_args.tensorizer_uri = args.path_to_tensors
204 + else:
205 + tensorizer_args = None
206
210 -keyfile = args.keyfile if args.keyfile else None
207 + if args.command == "serialize":
208 + eng_args_dict = {f.name: getattr(args, f.name) for f in
209 + dataclasses.fields(EngineArgs)}
210
211 + engine_args = EngineArgs.from_cli_args(
212 + argparse.Namespace(**eng_args_dict)
213 + )
214
213 -if args.model_loader_extra_config:
214 - config = json.loads(args.model_loader_extra_config)
215 - tensorizer_args = TensorizerConfig(**config)._construct_tensorizer_args()
216 - tensorizer_args.tensorizer_uri = args.path_to_tensors
217 -else:
218 - tensorizer_args = None
219 -
220 -if args.command == "serialize":
221 - eng_args_dict = {f.name: getattr(args, f.name) for f in
222 - dataclasses.fields(EngineArgs)}
223 -
224 - engine_args = EngineArgs.from_cli_args(argparse.Namespace(**eng_args_dict))
225 - engine = LLMEngine.from_engine_args(engine_args)
215 + input_dir = args.serialized_directory.rstrip('/')
216 + suffix = args.suffix if args.suffix else uuid.uuid4().hex
217 + base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
218 + if engine_args.tensor_parallel_size > 1:
219 + model_path = f"{base_path}/model-rank-%03d.tensors"
220 + else:
221 + model_path = f"{base_path}/model.tensors"
222
227 - input_dir = args.serialized_directory.rstrip('/')
228 - suffix = args.suffix if args.suffix else uuid.uuid4().hex
229 - base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
230 - model_path = f"{base_path}/model.tensors"
231 - tensorizer_config = TensorizerConfig(
232 - tensorizer_uri=model_path,
233 - **credentials)
234 - serialize_vllm_model(engine, tensorizer_config, keyfile)
235 -elif args.command == "deserialize":
236 - if not tensorizer_args:
223 tensorizer_config = TensorizerConfig(
238 - tensorizer_uri=args.path_to_tensors,
239 - encryption_keyfile = keyfile,
240 - **credentials
241 - )
242 - deserialize()
243 -else:
244 - raise ValueError("Either serialize or deserialize must be specified.")
224 + tensorizer_uri=model_path,
225 + encryption_keyfile=keyfile,
226 + **credentials)
227 +
228 + tensorize_vllm_model(engine_args, tensorizer_config)
229 +
230 + elif args.command == "deserialize":
231 + if not tensorizer_args:
232 + tensorizer_config = TensorizerConfig(
233 + tensorizer_uri=args.path_to_tensors,
234 + encryption_keyfile = keyfile,
235 + **credentials
236 + )
237 + deserialize()
238 + else:
239 + raise ValueError("Either serialize or deserialize must be specified.")
tests/tensorizer_loader/test_tensorizer.py
+90 -9
@@ -1,21 +1,27 @@
1 import json
2 import os
3 +import pathlib
4 import subprocess
5 from unittest.mock import MagicMock, patch
6
7 import openai
8 import pytest
9 import ray
10 +import torch
11 +from tensorizer import EncryptionParams
12
13 from vllm import SamplingParams
14 +from vllm.engine.arg_utils import EngineArgs
15 # yapf: disable
16 from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
17 TensorSerializer,
18 is_vllm_tensorized,
19 load_with_tensorizer,
20 open_stream,
17 - serialize_vllm_model)
21 + serialize_vllm_model,
22 + tensorize_vllm_model)
23
24 +from ..conftest import VllmRunner, cleanup
25 from ..utils import ServerRunner
26
27 # yapf conflicts with isort for this docstring
@@ -42,6 +48,20 @@ def is_curl_installed():
48 except (subprocess.CalledProcessError, FileNotFoundError):
49 return False
50
51 +def get_torch_model(vllm_runner: VllmRunner):
52 + return vllm_runner \
53 + .model \
54 + .llm_engine \
55 + .model_executor \
56 + .driver_worker \
57 + .model_runner \
58 + .model
59 +
60 +def write_keyfile(keyfile_path: str):
61 + encryption_params = EncryptionParams.random()
62 + pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
63 + with open(keyfile_path, 'wb') as f:
64 + f.write(encryption_params.key)
65
66 @pytest.fixture(autouse=True)
67 def tensorizer_config():
@@ -88,12 +108,17 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
108 with vllm_runner(model_ref) as vllm_model:
109 model_path = tmp_path / (model_ref + ".tensors")
110 key_path = tmp_path / (model_ref + ".key")
111 + write_keyfile(key_path)
112 +
113 outputs = vllm_model.generate(prompts, sampling_params)
114
93 - config_for_serializing = TensorizerConfig(tensorizer_uri=model_path)
94 - serialize_vllm_model(vllm_model.model.llm_engine,
95 - config_for_serializing,
96 - encryption_key_path=key_path)
115 + config_for_serializing = TensorizerConfig(
116 + tensorizer_uri=model_path,
117 + encryption_keyfile=key_path
118 + )
119 + serialize_vllm_model(get_torch_model(vllm_model),
120 + config_for_serializing)
121 +
122
123 config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
124 encryption_keyfile=key_path)
@@ -145,7 +170,7 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
170 with vllm_runner(model_ref, ) as vllm_model:
171 model_path = tmp_path / (model_ref + ".tensors")
172
148 - serialize_vllm_model(vllm_model.model.llm_engine,
173 + serialize_vllm_model(get_torch_model(vllm_model),
174 TensorizerConfig(tensorizer_uri=model_path))
175
176 with vllm_runner(
@@ -180,7 +205,7 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
205 with vllm_runner(model_ref, ) as vllm_model:
206 model_path = tmp_path / (model_ref + ".tensors")
207
183 - serialize_vllm_model(vllm_model.model.llm_engine,
208 + serialize_vllm_model(get_torch_model(vllm_model),
209 TensorizerConfig(tensorizer_uri=model_path))
210
211 model_loader_extra_config = {
@@ -224,7 +249,9 @@ def test_raise_value_error_on_invalid_load_format(vllm_runner):
249 model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"))
250
251
227 -def test_tensorizer_with_tp(vllm_runner):
252 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
253 + reason="Requires 2 GPUs")
254 +def test_tensorizer_with_tp_path_without_template(vllm_runner):
255 with pytest.raises(ValueError):
256 model_ref = "EleutherAI/pythia-1.4b"
257 tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
@@ -238,8 +265,62 @@ def test_tensorizer_with_tp(vllm_runner):
265 s3_endpoint="object.ord1.coreweave.com",
266 ),
267 tensor_parallel_size=2,
268 + disable_custom_all_reduce=True,
269 )
270
271 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
272 + reason="Requires 2 GPUs")
273 +def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
274 + tmp_path):
275 + model_ref = "EleutherAI/pythia-1.4b"
276 + # record outputs from un-sharded un-tensorized model
277 + base_model = vllm_runner(
278 + model_ref,
279 + disable_custom_all_reduce=True,
280 + enforce_eager=True,
281 + )
282 + outputs = base_model.generate(prompts, sampling_params)
283 +
284 + base_model.model.llm_engine.model_executor.shutdown()
285 + del base_model
286 + cleanup()
287 + ray.shutdown()
288 +
289 + # load model with two shards and serialize with encryption
290 + model_path = str(tmp_path / (model_ref + "-%02d.tensors"))
291 + key_path = tmp_path / (model_ref + ".key")
292 +
293 + tensorizer_config = TensorizerConfig(
294 + tensorizer_uri=model_path,
295 + encryption_keyfile=key_path,
296 + )
297 +
298 + tensorize_vllm_model(
299 + engine_args=EngineArgs(
300 + model=model_ref,
301 + tensor_parallel_size=2,
302 + disable_custom_all_reduce=True,
303 + enforce_eager=True,
304 + ),
305 + tensorizer_config=tensorizer_config,
306 + )
307 + assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
308 + assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
309 + cleanup()
310 + ray.shutdown()
311 +
312 + loaded_vllm_model = vllm_runner(
313 + model_ref,
314 + tensor_parallel_size=2,
315 + load_format="tensorizer",
316 + disable_custom_all_reduce=True,
317 + enforce_eager=True,
318 + model_loader_extra_config=tensorizer_config)
319 +
320 + deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
321 +
322 + assert outputs == deserialized_outputs
323 +
324
325 def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
326 model_ref = "facebook/opt-125m"
@@ -248,7 +329,7 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
329
330 with vllm_runner(model_ref) as vllm_model:
331 outputs = vllm_model.generate(prompts, sampling_params)
251 - serialize_vllm_model(vllm_model.model.llm_engine, config)
332 + serialize_vllm_model(get_torch_model(vllm_model), config)
333
334 assert is_vllm_tensorized(config)
335
vllm/model_executor/model_loader/loader.py
+17 -1
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.quantization.base_config import (
24 QuantizationConfig)
25 from vllm.model_executor.model_loader.tensorizer import (
26 TensorizerConfig, is_vllm_tensorized, load_with_tensorizer,
27 - tensorizer_weights_iterator)
27 + serialize_vllm_model, tensorizer_weights_iterator)
28 from vllm.model_executor.model_loader.utils import (get_model_architecture,
29 set_default_torch_dtype)
30 from vllm.model_executor.model_loader.weight_utils import (
@@ -392,6 +392,12 @@ class TensorizerLoader(BaseModelLoader):
392 cache_config: CacheConfig) -> nn.Module:
393 self._verify_config(model_config, parallel_config)
394
395 + if parallel_config.tensor_parallel_size > 1:
396 + from vllm.distributed import get_tensor_model_parallel_rank
397 + self.tensorizer_config.tensorizer_uri = \
398 + self.tensorizer_config.tensorizer_uri \
399 + % get_tensor_model_parallel_rank()
400 +
401 if is_vllm_tensorized(self.tensorizer_config):
402 return self._load_model_serialized(model_config, device_config,
403 lora_config,
@@ -402,6 +408,16 @@ class TensorizerLoader(BaseModelLoader):
408 vision_language_config,
409 cache_config)
410
411 + @staticmethod
412 + def save_model(
413 + model: torch.nn.Module,
414 + tensorizer_config: TensorizerConfig,
415 + ) -> None:
416 + serialize_vllm_model(
417 + model=model,
418 + tensorizer_config=tensorizer_config,
419 + )
420 +
421
422 class ShardedStateLoader(BaseModelLoader):
423 """
vllm/model_executor/model_loader/tensorizer.py
+75 -32
@@ -2,11 +2,11 @@ import argparse
2 import dataclasses
3 import io
4 import os
5 +import re
6 import time
6 -import typing
7 from dataclasses import dataclass
8 from functools import partial
9 -from typing import Generator, Optional, Tuple, Type, Union
9 +from typing import BinaryIO, Generator, Optional, Tuple, Type, Union
10
11 import torch
12 from torch import nn
@@ -14,6 +14,7 @@ from transformers import PretrainedConfig
14
15 import vllm.envs as envs
16 from vllm.config import ModelConfig, ParallelConfig
17 +from vllm.engine.arg_utils import EngineArgs
18 from vllm.engine.llm_engine import LLMEngine
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.quantization.base_config import (
@@ -48,8 +49,7 @@ logger = init_logger(__name__)
49
50 @dataclass
51 class TensorizerConfig:
51 - tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
52 - str, bytes, os.PathLike, int]
52 + tensorizer_uri: str
53 vllm_tensorized: Optional[bool] = False
54 verify_hash: Optional[bool] = False
55 num_readers: Optional[int] = None
@@ -60,6 +60,12 @@ class TensorizerConfig:
60 model_class: Optional[Type[torch.nn.Module]] = None
61 hf_config: Optional[PretrainedConfig] = None
62 dtype: Optional[Union[str, torch.dtype]] = None
63 + _is_sharded: bool = False
64 +
65 + def __post_init__(self):
66 + # check if the configuration is for a sharded vLLM model
67 + self._is_sharded = isinstance(self.tensorizer_uri, str) \
68 + and re.search(r'%0\dd', self.tensorizer_uri) is not None
69
70 def _construct_tensorizer_args(self) -> "TensorizerArgs":
71 tensorizer_args = {
@@ -78,13 +84,12 @@ class TensorizerConfig:
84 self,
85 parallel_config: "ParallelConfig",
86 ) -> None:
81 - if (parallel_config.tensor_parallel_size > 1
82 - and self.tensorizer_uri is not None):
87 + if parallel_config.tensor_parallel_size > 1 \
88 + and not self._is_sharded:
89 raise ValueError(
84 - "Loading to multiple GPUs is not currently supported with "
85 - "vLLM-serialized models. Please set tensor_parallel_size=1."
86 - " or use a non-vLLM-serialized model, such as a "
87 - "serialized Hugging Face `PretrainedModel`.")
90 + "For a sharded model, tensorizer_uri should include a"
91 + " string format template like '%04d' to be formatted"
92 + " with the rank of the shard")
93
94 def verify_with_model_config(self, model_config: "ModelConfig") -> None:
95 if (model_config.quantization is not None
@@ -102,8 +107,8 @@ def load_with_tensorizer(tensorizer_config: TensorizerConfig,
107
108 @dataclass
109 class TensorizerArgs:
105 - tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
106 - str, bytes, os.PathLike, int]
110 + tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, BinaryIO, str,
111 + bytes, os.PathLike, int]
112 vllm_tensorized: Optional[bool] = False
113 verify_hash: Optional[bool] = False
114 num_readers: Optional[int] = None
@@ -332,6 +337,7 @@ class TensorizerAgent:
337 ) as stream, TensorDeserializer(
338 stream,
339 dtype=self.tensorizer_config.dtype,
340 + device=f'cuda:{torch.cuda.current_device()}',
341 **self.tensorizer_args.deserializer_params) as deserializer:
342 deserializer.load_into_module(self.model)
343 end = time.perf_counter()
@@ -400,33 +406,70 @@ def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
406 return False
407
408
403 -def get_pretensorized_vllm_model(engine: "LLMEngine") -> nn.Module:
404 - model = (engine.model_executor.driver_worker.model_runner.model)
409 +def serialize_vllm_model(
410 + model: nn.Module,
411 + tensorizer_config: TensorizerConfig,
412 +) -> nn.Module:
413 model.register_parameter(
414 "vllm_tensorized_marker",
415 nn.Parameter(torch.tensor((1, ), device="meta"), requires_grad=False))
408 - return model
409 -
410 -
411 -def serialize_vllm_model(engine: "LLMEngine",
412 - tensorizer_config : TensorizerConfig,
413 - encryption_key_path: Optional[str] = None) \
414 - -> nn.Module:
415 -
416 - model = get_pretensorized_vllm_model(engine)
416 tensorizer_args = tensorizer_config._construct_tensorizer_args()
417 +
418 encryption_params = None
419 - if encryption_key_path is not None:
420 - encryption_params = EncryptionParams.random()
421 - with _write_stream(encryption_key_path,
422 - **tensorizer_args.stream_params) as stream:
423 - stream.write(encryption_params.key)
419 + if (keyfile := tensorizer_config.encryption_keyfile) is not None:
420 + with open(keyfile, "rb") as f:
421 + key = f.read()
422 + encryption_params = EncryptionParams(key=key)
423
425 - with _write_stream(tensorizer_args.tensorizer_uri,
426 - **tensorizer_args.stream_params) as stream:
424 + output_file = tensorizer_args.tensorizer_uri
425 + if tensorizer_config._is_sharded:
426 + from vllm.distributed import get_tensor_model_parallel_rank
427 + output_file = output_file % get_tensor_model_parallel_rank()
428 +
429 + with _write_stream(output_file, **tensorizer_args.stream_params) as stream:
430 serializer = TensorSerializer(stream, encryption=encryption_params)
431 serializer.write_module(model)
432 serializer.close()
430 - logger.info("Successfully serialized model to %s",
431 - str(tensorizer_args.tensorizer_uri))
433 + logger.info("Successfully serialized model to %s", str(output_file))
434 return model
435 +
436 +
437 +def tensorize_vllm_model(engine_args: EngineArgs,
438 + tensorizer_config: TensorizerConfig,
439 + generate_keyfile: bool = True):
440 + """Utility to load a model and then serialize it with Tensorizer
441 +
442 + Intended to be used separately from running a vLLM server since it
443 + creates its own Engine instance.
444 + """
445 + engine_config = engine_args.create_engine_config()
446 + tensorizer_config.verify_with_model_config(engine_config.model_config)
447 + tensorizer_config.verify_with_parallel_config(
448 + engine_config.parallel_config)
449 +
450 + # generate the encryption key before creating the engine to support sharding
451 + if generate_keyfile and (keyfile :=
452 + tensorizer_config.encryption_keyfile) is not None:
453 + encryption_params = EncryptionParams.random()
454 + with _write_stream(
455 + keyfile,
456 + s3_access_key_id=tensorizer_config.s3_access_key_id,
457 + s3_secret_access_key=tensorizer_config.s3_secret_access_key,
458 + s3_endpoint=tensorizer_config.s3_endpoint,
459 + ) as stream:
460 + stream.write(encryption_params.key)
461 +
462 + engine = LLMEngine.from_engine_args(engine_args)
463 + if tensorizer_config._is_sharded:
464 + # if the engine is a distributed engine (for tensor parallel) then each
465 + # worker shard needs to serialize its part of the model.
466 + engine.model_executor._run_workers(
467 + "save_tensorized_model",
468 + tensorizer_config=tensorizer_config,
469 + )
470 + else:
471 + # with a single worker, we can get to the underlying model directly
472 + serialize_vllm_model(
473 + engine.model_executor.driver_worker.model_runner.model,
474 + tensorizer_config,
475 + )
vllm/worker/model_runner.py
+11
@@ -20,6 +20,7 @@ from vllm.lora.request import LoRARequest
20 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
21 from vllm.model_executor import SamplingMetadata
22 from vllm.model_executor.model_loader import get_model
23 +from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.sampling_params import SamplingParams
26 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
@@ -222,6 +223,16 @@ class ModelRunner:
223 max_size=max_size,
224 )
225
226 + def save_tensorized_model(
227 + self,
228 + tensorizer_config: TensorizerConfig,
229 + ) -> None:
230 + from vllm.model_executor.model_loader.loader import TensorizerLoader
231 + TensorizerLoader.save_model(
232 + self.model,
233 + tensorizer_config=tensorizer_config,
234 + )
235 +
236 def get_max_block_per_batch(self) -> int:
237 block_size = self.block_size
238 return (self.max_seq_len_to_capture + block_size - 1) // block_size
vllm/worker/worker.py
+8
@@ -15,6 +15,7 @@ from vllm.distributed import (broadcast_tensor_dict,
15 set_custom_all_reduce)
16 from vllm.lora.request import LoRARequest
17 from vllm.model_executor import set_random_seed
18 +from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19 from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
20 from vllm.worker.cache_engine import CacheEngine
21 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
@@ -132,6 +133,13 @@ class Worker(WorkerBase):
133 max_size=max_size,
134 )
135
136 + def save_tensorized_model(
137 + self,
138 + tensorizer_config: TensorizerConfig,
139 + ) -> None:
140 + self.model_runner.save_tensorized_model(
141 + tensorizer_config=tensorizer_config, )
142 +
143 @torch.inference_mode()
144 def determine_num_available_blocks(self) -> Tuple[int, int]:
145 """Profiles the peak memory usage of the model to determine how many