@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 995f56236b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Core] Loading model from S3 using RunAI Model Streamer as optional loader (#10192)

Signed-off-by: OmerD <omer@run.ai>

omer-dayan committed Dec 20, 2024 at 18:46 UTC 995f56236bc08300ea11fc8cd3d66029ffec8678
13 files changed +457 -3
Dockerfile
+2 -2
@@ -240,9 +240,9 @@ FROM vllm-base AS vllm-openai
240 # install additional dependencies for openai api server
241 RUN --mount=type=cache,target=/root/.cache/pip \
242 if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
243 - pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10'; \
243 + pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
244 else \
245 - pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10'; \
245 + pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
246 fi
247
248 ENV VLLM_USAGE_SOURCE production-docker-image
docs/source/index.rst
+1
@@ -88,6 +88,7 @@ Documentation
88 serving/metrics
89 serving/integrations
90 serving/tensorizer
91 + serving/runai_model_streamer
92
93 .. toctree::
94 :maxdepth: 1
docs/source/serving/runai_model_streamer.rst new
+53
@@ -0,0 +1,53 @@
1 +.. _runai_model_streamer:
2 +
3 +Loading Models with Run:ai Model Streamer
4 +=========================================
5 +Run:ai Model Streamer is a library to read tensors in concurrency, while streaming it to GPU memory.
6 +Further reading can be found in `Run:ai Model Streamer Documentation <https://github.com/run-ai/runai-model-streamer/blob/master/docs/README.md>`_.
7 +
8 +vLLM supports loading weights in Safetensors format using the Run:ai Model Streamer.
9 +You first need to install vLLM RunAI optional dependency:
10 +
11 +.. code-block:: console
12 +
13 + $ pip3 install vllm[runai]
14 +
15 +To run it as an OpenAI-compatible server, add the `--load-format runai_streamer` flag:
16 +
17 +.. code-block:: console
18 +
19 + $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer
20 +
21 +To run model from AWS S3 object store run:
22 +
23 +.. code-block:: console
24 +
25 + $ vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
26 +
27 +
28 +To run model from a S3 compatible object store run:
29 +
30 +.. code-block:: console
31 +
32 + $ RUNAI_STREAMER_S3_USE_VIRTUAL_ADDRESSING=0 AWS_EC2_METADATA_DISABLED=true AWS_ENDPOINT_URL=https://storage.googleapis.com vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
33 +
34 +Tunable parameters
35 +------------------
36 +You can tune parameters using `--model-loader-extra-config`:
37 +
38 +You can tune `concurrency` that controls the level of concurrency and number of OS threads reading tensors from the file to the CPU buffer.
39 +For reading from S3, it will be the number of client instances the host is opening to the S3 server.
40 +
41 + .. code-block:: console
42 +
43 + $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"concurrency":16}'
44 +
45 +You can controls the size of the CPU Memory buffer to which tensors are read from the file, and limit this size.
46 +You can read further about CPU buffer memory limiting `here <https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md#runai_streamer_memory_limit>`_.
47 +
48 + .. code-block:: console
49 +
50 + $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"memory_limit":5368709120}'
51 +
52 +.. note::
53 + For further instructions about tunable parameters and additional parameters configurable through environment variables, read the `Environment Variables Documentation <https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md>`_.
setup.py
+1
@@ -630,6 +630,7 @@ setup(
630 ext_modules=ext_modules,
631 extras_require={
632 "tensorizer": ["tensorizer>=2.9.0"],
633 + "runai": ["runai-model-streamer", "runai-model-streamer-s3", "boto3"],
634 "audio": ["librosa", "soundfile"], # Required for audio processing
635 "video": ["decord"] # Required for video processing
636 },
tests/runai_model_streamer/__init__.py
tests/runai_model_streamer/test_runai_model_streamer_loader.py new
+31
@@ -0,0 +1,31 @@
1 +from vllm import SamplingParams
2 +from vllm.config import LoadConfig, LoadFormat
3 +from vllm.model_executor.model_loader.loader import (RunaiModelStreamerLoader,
4 + get_model_loader)
5 +
6 +test_model = "openai-community/gpt2"
7 +
8 +prompts = [
9 + "Hello, my name is",
10 + "The president of the United States is",
11 + "The capital of France is",
12 + "The future of AI is",
13 +]
14 +# Create a sampling params object.
15 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
16 +
17 +
18 +def get_runai_model_loader():
19 + load_config = LoadConfig(load_format=LoadFormat.RUNAI_STREAMER)
20 + return get_model_loader(load_config)
21 +
22 +
23 +def test_get_model_loader_with_runai_flag():
24 + model_loader = get_runai_model_loader()
25 + assert isinstance(model_loader, RunaiModelStreamerLoader)
26 +
27 +
28 +def test_runai_model_loader_download_files(vllm_runner):
29 + with vllm_runner(test_model, load_format=LoadFormat.RUNAI_STREAMER) as llm:
30 + deserialized_outputs = llm.generate(prompts, sampling_params)
31 + assert deserialized_outputs
tests/runai_model_streamer/test_weight_utils.py new
+39
@@ -0,0 +1,39 @@
1 +import glob
2 +import tempfile
3 +
4 +import huggingface_hub.constants
5 +import torch
6 +
7 +from vllm.model_executor.model_loader.weight_utils import (
8 + download_weights_from_hf, runai_safetensors_weights_iterator,
9 + safetensors_weights_iterator)
10 +
11 +
12 +def test_runai_model_loader():
13 + with tempfile.TemporaryDirectory() as tmpdir:
14 + huggingface_hub.constants.HF_HUB_OFFLINE = False
15 + download_weights_from_hf("openai-community/gpt2",
16 + allow_patterns=["*.safetensors"],
17 + cache_dir=tmpdir)
18 + safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
19 + assert len(safetensors) > 0
20 +
21 + runai_model_streamer_tensors = {}
22 + hf_safetensors_tensors = {}
23 +
24 + for name, tensor in runai_safetensors_weights_iterator(safetensors):
25 + runai_model_streamer_tensors[name] = tensor
26 +
27 + for name, tensor in safetensors_weights_iterator(safetensors):
28 + hf_safetensors_tensors[name] = tensor
29 +
30 + assert len(runai_model_streamer_tensors) == len(hf_safetensors_tensors)
31 +
32 + for name, runai_tensor in runai_model_streamer_tensors.items():
33 + assert runai_tensor.dtype == hf_safetensors_tensors[name].dtype
34 + assert runai_tensor.shape == hf_safetensors_tensors[name].shape
35 + assert torch.all(runai_tensor.eq(hf_safetensors_tensors[name]))
36 +
37 +
38 +if __name__ == "__main__":
39 + test_runai_model_loader()
vllm/config.py
+37
@@ -29,6 +29,7 @@ from vllm.transformers_utils.config import (
29 get_hf_text_config, get_pooling_config,
30 get_sentence_transformer_tokenizer_config, is_encoder_decoder,
31 try_get_generation_config, uses_mrope)
32 +from vllm.transformers_utils.utils import is_s3
33 from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
34 get_cpu_memory, print_warning_once, random_uuid,
35 resolve_obj_by_qualname)
@@ -256,6 +257,8 @@ class ModelConfig:
257 f"'Please instead use `--hf-overrides '{hf_override!r}'`")
258 warnings.warn(DeprecationWarning(msg), stacklevel=2)
259
260 + self.maybe_pull_model_tokenizer_for_s3(model, tokenizer)
261 +
262 # The tokenizer version is consistent with the model version by default.
263 if tokenizer_revision is None:
264 self.tokenizer_revision = revision
@@ -357,6 +360,39 @@ class ModelConfig:
360 self._verify_cuda_graph()
361 self._verify_bnb_config()
362
363 + def maybe_pull_model_tokenizer_for_s3(self, model: str,
364 + tokenizer: str) -> None:
365 + """
366 + Pull the model config or tokenizer to a temporary
367 + directory in case of S3.
368 +
369 + Args:
370 + model: The model name or path.
371 + tokenizer: The tokenizer name or path.
372 +
373 + """
374 + if is_s3(model) or is_s3(tokenizer):
375 + try:
376 + from vllm.transformers_utils.s3_utils import S3Model
377 + except ImportError as err:
378 + raise ImportError(
379 + "Please install Run:ai optional dependency "
380 + "to use the S3 capabilities. "
381 + "You can install it with: pip install vllm[runai]"
382 + ) from err
383 +
384 + if is_s3(model):
385 + self.s3_model = S3Model()
386 + self.s3_model.pull_files(model, allow_pattern=["*config.json"])
387 + self.model_weights = self.model
388 + self.model = self.s3_model.dir
389 +
390 + if is_s3(tokenizer):
391 + self.s3_tokenizer = S3Model()
392 + self.s3_tokenizer.pull_files(
393 + model, ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
394 + self.tokenizer = self.s3_tokenizer.dir
395 +
396 def _init_multimodal_config(
397 self, limit_mm_per_prompt: Optional[Mapping[str, int]]
398 ) -> Optional["MultiModalConfig"]:
@@ -1099,6 +1135,7 @@ class LoadFormat(str, enum.Enum):
1135 GGUF = "gguf"
1136 BITSANDBYTES = "bitsandbytes"
1137 MISTRAL = "mistral"
1138 + RUNAI_STREAMER = "runai_streamer"
1139
1140
1141 @dataclass
vllm/engine/arg_utils.py
+2
@@ -316,6 +316,8 @@ class EngineArgs:
316 '* "tensorizer" will load the weights using tensorizer from '
317 'CoreWeave. See the Tensorize vLLM Model script in the Examples '
318 'section for more information.\n'
319 + '* "runai_streamer" will load the Safetensors weights using Run:ai'
320 + 'Model Streamer \n'
321 '* "bitsandbytes" will load the weights using bitsandbytes '
322 'quantization.\n')
323 parser.add_argument(
vllm/model_executor/model_loader/loader.py
+117 -1
@@ -45,9 +45,10 @@ from vllm.model_executor.model_loader.weight_utils import (
45 filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
46 get_gguf_extra_tensor_names, gguf_quant_weights_iterator,
47 initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
48 - safetensors_weights_iterator)
48 + runai_safetensors_weights_iterator, safetensors_weights_iterator)
49 from vllm.model_executor.utils import set_weight_attrs
50 from vllm.platforms import current_platform
51 +from vllm.transformers_utils.utils import is_s3
52 from vllm.utils import is_pin_memory_available
53
54
@@ -1234,6 +1235,118 @@ class GGUFModelLoader(BaseModelLoader):
1235 return model
1236
1237
1238 +class RunaiModelStreamerLoader(BaseModelLoader):
1239 + """
1240 + Model loader that can load safetensors
1241 + files from local FS or S3 bucket.
1242 + """
1243 +
1244 + def __init__(self, load_config: LoadConfig):
1245 + super().__init__(load_config)
1246 + if load_config.model_loader_extra_config:
1247 + extra_config = load_config.model_loader_extra_config
1248 +
1249 + if ("concurrency" in extra_config
1250 + and isinstance(extra_config.get("concurrency"), int)):
1251 + os.environ["RUNAI_STREAMER_CONCURRENCY"] = str(
1252 + extra_config.get("concurrency"))
1253 +
1254 + if ("memory_limit" in extra_config
1255 + and isinstance(extra_config.get("memory_limit"), int)):
1256 + os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] = str(
1257 + extra_config.get("memory_limit"))
1258 +
1259 + runai_streamer_s3_endpoint = os.getenv(
1260 + 'RUNAI_STREAMER_S3_ENDPOINT')
1261 + aws_endpoint_url = os.getenv('AWS_ENDPOINT_URL')
1262 + if (runai_streamer_s3_endpoint is None
1263 + and aws_endpoint_url is not None):
1264 + os.environ["RUNAI_STREAMER_S3_ENDPOINT"] = aws_endpoint_url
1265 +
1266 + def _prepare_weights(self, model_name_or_path: str,
1267 + revision: Optional[str]) -> List[str]:
1268 + """Prepare weights for the model.
1269 +
1270 + If the model is not local, it will be downloaded."""
1271 + is_s3_path = is_s3(model_name_or_path)
1272 + if is_s3_path:
1273 + try:
1274 + from vllm.transformers_utils.s3_utils import glob as s3_glob
1275 + except ImportError as err:
1276 + raise ImportError(
1277 + "Please install Run:ai optional dependency "
1278 + "to use the S3 capabilities. "
1279 + "You can install it with: pip install vllm[runai]"
1280 + ) from err
1281 +
1282 + is_local = os.path.isdir(model_name_or_path)
1283 + safetensors_pattern = "*.safetensors"
1284 + index_file = SAFE_WEIGHTS_INDEX_NAME
1285 +
1286 + hf_folder = (model_name_or_path if
1287 + (is_local or is_s3_path) else download_weights_from_hf(
1288 + model_name_or_path,
1289 + self.load_config.download_dir,
1290 + [safetensors_pattern],
1291 + revision,
1292 + ignore_patterns=self.load_config.ignore_patterns,
1293 + ))
1294 +
1295 + if is_s3_path:
1296 + hf_weights_files = s3_glob(path=hf_folder,
1297 + allow_pattern=[safetensors_pattern])
1298 + else:
1299 + hf_weights_files = glob.glob(
1300 + os.path.join(hf_folder, safetensors_pattern))
1301 +
1302 + if not is_local and not is_s3_path:
1303 + download_safetensors_index_file_from_hf(
1304 + model_name_or_path, index_file, self.load_config.download_dir,
1305 + revision)
1306 +
1307 + if not hf_weights_files:
1308 + raise RuntimeError(
1309 + f"Cannot find any safetensors model weights with "
1310 + f"`{model_name_or_path}`")
1311 +
1312 + return hf_weights_files
1313 +
1314 + def _get_weights_iterator(
1315 + self, model_or_path: str,
1316 + revision: str) -> Generator[Tuple[str, torch.Tensor], None, None]:
1317 + """Get an iterator for the model weights based on the load format."""
1318 + hf_weights_files = self._prepare_weights(model_or_path, revision)
1319 + return runai_safetensors_weights_iterator(hf_weights_files)
1320 +
1321 + def download_model(self, model_config: ModelConfig) -> None:
1322 + """Download model if necessary"""
1323 + self._prepare_weights(model_config.model, model_config.revision)
1324 +
1325 + def load_model(self, vllm_config: VllmConfig) -> nn.Module:
1326 + """Perform streaming of the model to destination"""
1327 + device_config = vllm_config.device_config
1328 + model_config = vllm_config.model_config
1329 +
1330 + target_device = torch.device(device_config.device)
1331 + with set_default_torch_dtype(model_config.dtype):
1332 + with target_device:
1333 + model = _initialize_model(vllm_config=vllm_config)
1334 +
1335 + model_weights = model_config.model
1336 + if hasattr(model_config, "model_weights"):
1337 + model_weights = model_config.model_weights
1338 + model.load_weights(
1339 + self._get_weights_iterator(model_weights,
1340 + model_config.revision))
1341 +
1342 + for _, module in model.named_modules():
1343 + quant_method = getattr(module, "quant_method", None)
1344 + if quant_method is not None:
1345 + with device_loading_context(module, target_device):
1346 + quant_method.process_weights_after_loading(module)
1347 + return model.eval()
1348 +
1349 +
1350 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1351 """Get a model loader based on the load format."""
1352
@@ -1255,4 +1368,7 @@ def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1368 if load_config.load_format == LoadFormat.GGUF:
1369 return GGUFModelLoader(load_config)
1370
1371 + if load_config.load_format == LoadFormat.RUNAI_STREAMER:
1372 + return RunaiModelStreamerLoader(load_config)
1373 +
1374 return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/weight_utils.py
+24
@@ -410,6 +410,30 @@ def safetensors_weights_iterator(
410 yield name, param
411
412
413 +def runai_safetensors_weights_iterator(
414 + hf_weights_files: List[str]
415 +) -> Generator[Tuple[str, torch.Tensor], None, None]:
416 + """Iterate over the weights in the model safetensor files."""
417 + try:
418 + from runai_model_streamer import SafetensorsStreamer
419 + except ImportError as err:
420 + raise ImportError(
421 + "Please install Run:ai optional dependency."
422 + "You can install it with: pip install vllm[runai]") from err
423 +
424 + enable_tqdm = not torch.distributed.is_initialized(
425 + ) or torch.distributed.get_rank() == 0
426 + with SafetensorsStreamer() as streamer:
427 + for st_file in tqdm(
428 + hf_weights_files,
429 + desc="Loading safetensors using Runai Model Streamer",
430 + disable=not enable_tqdm,
431 + bar_format=_BAR_FORMAT,
432 + ):
433 + streamer.stream_file(st_file)
434 + yield from streamer.get_tensors()
435 +
436 +
437 def pt_weights_iterator(
438 hf_weights_files: List[str]
439 ) -> Generator[Tuple[str, torch.Tensor], None, None]:
vllm/transformers_utils/s3_utils.py new
+146
@@ -0,0 +1,146 @@
1 +import fnmatch
2 +import os
3 +import shutil
4 +import signal
5 +import tempfile
6 +from pathlib import Path
7 +from typing import Optional
8 +
9 +import boto3
10 +
11 +
12 +def _filter_allow(paths: list[str], patterns: list[str]) -> list[str]:
13 + return [
14 + path for path in paths if any(
15 + fnmatch.fnmatch(path, pattern) for pattern in patterns)
16 + ]
17 +
18 +
19 +def _filter_ignore(paths: list[str], patterns: list[str]) -> list[str]:
20 + return [
21 + path for path in paths
22 + if not any(fnmatch.fnmatch(path, pattern) for pattern in patterns)
23 + ]
24 +
25 +
26 +def glob(s3=None,
27 + path: str = "",
28 + allow_pattern: Optional[list[str]] = None) -> list[str]:
29 + """
30 + List full file names from S3 path and filter by allow pattern.
31 +
32 + Args:
33 + s3: S3 client to use.
34 + path: The S3 path to list from.
35 + allow_pattern: A list of patterns of which files to pull.
36 +
37 + Returns:
38 + list[str]: List of full S3 paths allowed by the pattern
39 + """
40 + if s3 is None:
41 + s3 = boto3.client("s3")
42 + bucket_name, _, paths = list_files(s3,
43 + path=path,
44 + allow_pattern=allow_pattern)
45 + return [f"s3://{bucket_name}/{path}" for path in paths]
46 +
47 +
48 +def list_files(
49 + s3,
50 + path: str,
51 + allow_pattern: Optional[list[str]] = None,
52 + ignore_pattern: Optional[list[str]] = None
53 +) -> tuple[str, str, list[str]]:
54 + """
55 + List files from S3 path and filter by pattern.
56 +
57 + Args:
58 + s3: S3 client to use.
59 + path: The S3 path to list from.
60 + allow_pattern: A list of patterns of which files to pull.
61 + ignore_pattern: A list of patterns of which files not to pull.
62 +
63 + Returns:
64 + tuple[str, str, list[str]]: A tuple where:
65 + - The first element is the bucket name
66 + - The second element is string represent the bucket
67 + and the prefix as a dir like string
68 + - The third element is a list of files allowed or
69 + disallowed by pattern
70 + """
71 + parts = path.removeprefix('s3://').split('/')
72 + prefix = '/'.join(parts[1:])
73 + bucket_name = parts[0]
74 +
75 + objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
76 + paths = [obj['Key'] for obj in objects.get('Contents', [])]
77 +
78 + paths = _filter_ignore(paths, ["*/"])
79 + if allow_pattern is not None:
80 + paths = _filter_allow(paths, allow_pattern)
81 +
82 + if ignore_pattern is not None:
83 + paths = _filter_ignore(paths, ignore_pattern)
84 +
85 + return bucket_name, prefix, paths
86 +
87 +
88 +class S3Model:
89 + """
90 + A class representing a S3 model mirrored into a temporary directory.
91 +
92 + Attributes:
93 + s3: S3 client.
94 + dir: The temporary created directory.
95 +
96 + Methods:
97 + pull_files(): Pull model from S3 to the temporary directory.
98 + """
99 +
100 + def __init__(self) -> None:
101 + self.s3 = boto3.client('s3')
102 + for sig in (signal.SIGINT, signal.SIGTERM):
103 + existing_handler = signal.getsignal(sig)
104 + signal.signal(sig, self._close_by_signal(existing_handler))
105 + self.dir = tempfile.mkdtemp()
106 +
107 + def __del__(self):
108 + self._close()
109 +
110 + def _close(self) -> None:
111 + if os.path.exists(self.dir):
112 + shutil.rmtree(self.dir)
113 +
114 + def _close_by_signal(self, existing_handler=None):
115 +
116 + def new_handler(signum, frame):
117 + self._close()
118 + if existing_handler:
119 + existing_handler(signum, frame)
120 +
121 + return new_handler
122 +
123 + def pull_files(self,
124 + s3_model_path: str = "",
125 + allow_pattern: Optional[list[str]] = None,
126 + ignore_pattern: Optional[list[str]] = None) -> None:
127 + """
128 + Pull files from S3 storage into the temporary directory.
129 +
130 + Args:
131 + s3_model_path: The S3 path of the model.
132 + allow_pattern: A list of patterns of which files to pull.
133 + ignore_pattern: A list of patterns of which files not to pull.
134 +
135 + """
136 + bucket_name, base_dir, files = list_files(self.s3, s3_model_path,
137 + allow_pattern,
138 + ignore_pattern)
139 + if len(files) == 0:
140 + return
141 +
142 + for file in files:
143 + destination_file = self.dir + file.removeprefix(base_dir)
144 + local_dir = Path(destination_file).parent
145 + os.makedirs(local_dir, exist_ok=True)
146 + self.s3.download_file(bucket_name, file, destination_file)
vllm/transformers_utils/utils.py
+4
@@ -3,6 +3,10 @@ from pathlib import Path
3 from typing import Union
4
5
6 +def is_s3(model_or_path: str) -> bool:
7 + return model_or_path.lower().startswith('s3://')
8 +
9 +
10 def check_gguf_file(model: Union[str, PathLike]) -> bool:
11 """Check if the file is a GGUF model."""
12 model = Path(model)