@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 69e1d2fb69
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Refactor model loading code (#4097)

Antoni Baum committed Apr 16, 2024 at 11:34 UTC 69e1d2fb6922b2d388bae41286d8867976cbd6c6
67 files changed +1064 -973
.buildkite/test-pipeline.yaml
+1 -1
@@ -92,7 +92,7 @@ steps:
92 parallelism: 4
93
94 - label: Tensorizer Test
95 - command: apt-get install curl libsodium23 && pytest -v -s tensorizer
95 + command: apt-get install curl libsodium23 && pytest -v -s tensorizer_loader
96
97 - label: Metrics Test
98 command: pytest -v -s metrics
examples/fp8/extract_scales.py
+2 -2
@@ -11,7 +11,7 @@ from safetensors.torch import safe_open
11 from vllm.model_executor.layers.quantization.schema import QuantParamSchema
12
13
14 -# Adapted from vllm/model_executor/weight_utils.py
14 +# Adapted from vllm/model_executor/model_loader/weight_utils.py
15 # The main differences are that we add the NPZ format and simplify
16 # its functionality drastically for our purposes (e.g. we assume that
17 # the quantized model exists locally and there is no need to download it)
@@ -71,7 +71,7 @@ def _prepare_hf_weights(
71 return hf_weights_files, use_safetensors
72
73
74 -# Adapted from vllm/model_executor/weight_utils.py
74 +# Adapted from vllm/model_executor/model_loader/weight_utils.py
75 def _hf_tensorfile_iterator(filename: str, load_format: str,
76 use_safetensors: bool):
77 if load_format == "npz":
examples/tensorize_vllm_model.py
+1 -1
@@ -16,8 +16,8 @@ from transformers import AutoConfig, PretrainedConfig
16 from vllm.distributed import initialize_model_parallel
17 from vllm.engine.arg_utils import EngineArgs
18 from vllm.engine.llm_engine import LLMEngine
19 +from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20 from vllm.model_executor.models import ModelRegistry
20 -from vllm.model_executor.tensorizer_loader import TensorizerArgs
21
22 # yapf conflicts with isort for this docstring
23 # yapf: disable
tests/lora/conftest.py
+5 -5
@@ -153,11 +153,11 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
153 cleanup()
154 get_model_old = get_model
155
156 - def get_model_patched(model_config, device_config, **kwargs):
157 - return get_model_old(model_config,
158 - device_config,
159 - lora_config=LoRAConfig(max_loras=4,
160 - max_lora_rank=8))
156 + def get_model_patched(*, model_config, device_config, **kwargs):
157 + kwargs["lora_config"] = LoRAConfig(max_loras=4, max_lora_rank=8)
158 + return get_model_old(model_config=model_config,
159 + device_config=device_config,
160 + **kwargs)
161
162 with patch("vllm.worker.model_runner.get_model", get_model_patched):
163 engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
tests/lora/test_worker.py
+6 -4
@@ -3,8 +3,8 @@ import random
3 import tempfile
4 from unittest.mock import patch
5
6 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7 - ParallelConfig, SchedulerConfig)
6 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 + ModelConfig, ParallelConfig, SchedulerConfig)
8 from vllm.lora.models import LoRAMapping
9 from vllm.lora.request import LoRARequest
10 from vllm.worker.worker import Worker
@@ -18,12 +18,14 @@ def test_worker_apply_lora(sql_lora_files):
18 "meta-llama/Llama-2-7b-hf",
19 tokenizer_mode="auto",
20 trust_remote_code=False,
21 - download_dir=None,
22 - load_format="dummy",
21 seed=0,
22 dtype="float16",
23 revision=None,
24 ),
25 + load_config=LoadConfig(
26 + download_dir=None,
27 + load_format="dummy",
28 + ),
29 parallel_config=ParallelConfig(1, 1, False),
30 scheduler_config=SchedulerConfig(32, 32, 32),
31 device_config=DeviceConfig("cuda"),
tests/model_executor/weight_utils.py
+1 -1
@@ -3,7 +3,7 @@ import os
3 import huggingface_hub.constants
4 import pytest
5
6 -from vllm.model_executor.weight_utils import enable_hf_transfer
6 +from vllm.model_executor.model_loader.weight_utils import enable_hf_transfer
7
8
9 def test_hf_transfer_auto_activation():
tests/quantization/test_autogptq_marlin_configs.py
-4
@@ -36,8 +36,6 @@ def test_auto_gptq(model_quant_type: str, ) -> None:
36 model_path,
37 tokenizer_mode="auto",
38 trust_remote_code=False,
39 - download_dir=None,
40 - load_format="dummy",
39 seed=0,
40 dtype="float16",
41 revision=None,
@@ -49,8 +47,6 @@ def test_auto_gptq(model_quant_type: str, ) -> None:
47 model_path,
48 tokenizer_mode="auto",
49 trust_remote_code=False,
52 - download_dir=None,
53 - load_format="dummy",
50 seed=0,
51 dtype="float16",
52 revision=None,
tests/samplers/test_sampler.py
+12 -2
@@ -32,7 +32,12 @@ def _prepare_test(
32 1e-2,
33 dtype=input_tensor.dtype)
34 sampler = MockLogitsSampler(fake_logits)
35 - model_runner = ModelRunner(None, None, None, None, None)
35 + model_runner = ModelRunner(model_config=None,
36 + parallel_config=None,
37 + scheduler_config=None,
38 + device_config=None,
39 + load_config=None,
40 + lora_config=None)
41 return input_tensor, fake_logits, sampler, model_runner
42
43
@@ -591,7 +596,12 @@ def test_sampler_top_k_top_p(seed: int, device: str):
596 device=input_tensor.device,
597 dtype=input_tensor.dtype)
598 sampler = MockLogitsSampler(fake_logits)
594 - model_runner = ModelRunner(None, None, None, None, None)
599 + model_runner = ModelRunner(model_config=None,
600 + parallel_config=None,
601 + scheduler_config=None,
602 + device_config=None,
603 + load_config=None,
604 + lora_config=None)
605
606 generation_model = GenerationMixin()
607 generation_config = GenerationConfig(top_k=top_k,
tests/spec_decode/utils.py
+1
@@ -118,6 +118,7 @@ def create_worker(cls: type,
118 scheduler_config=engine_config.scheduler_config,
119 device_config=engine_config.device_config,
120 cache_config=engine_config.cache_config,
121 + load_config=engine_config.load_config,
122 local_rank=0,
123 rank=0,
124 distributed_init_method=distributed_init_method,
tests/tensorizer_loader/__init__.py renamed
tests/tensorizer_loader/tensorize_vllm_model_for_testing.py renamed
+2 -2
@@ -16,8 +16,8 @@ from transformers import AutoConfig, PretrainedConfig
16 from vllm.distributed import initialize_model_parallel
17 from vllm.engine.arg_utils import EngineArgs
18 from vllm.engine.llm_engine import LLMEngine
19 +from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20 from vllm.model_executor.models import ModelRegistry
20 -from vllm.model_executor.tensorizer_loader import TensorizerArgs
21
22 # yapf conflicts with isort for this docstring
23 # yapf: disable
@@ -74,7 +74,7 @@ def parse_args():
74 "extremely quickly. Tensor encryption and decryption is "
75 "also supported, although libsodium must be installed to "
76 "use it.")
77 - parser = EngineArgs.add_cli_args(parser)
77 + parser = TensorizerArgs.add_cli_args(EngineArgs.add_cli_args(parser))
78 subparsers = parser.add_subparsers(dest='command')
79
80 serialize_parser = subparsers.add_parser(
tests/tensorizer_loader/test_tensorizer.py renamed
+82 -57
@@ -1,16 +1,19 @@
1 import gc
2 +import json
3 +import os
4 import subprocess
5 from unittest.mock import MagicMock, patch
6
7 +import openai
8 import pytest
9 +import ray
10 import torch
11
12 from tests.entrypoints.test_openai_server import ServerRunner
13 from vllm import SamplingParams
10 -from vllm.config import TensorizerConfig
11 -from vllm.model_executor.tensorizer_loader import (
12 - EncryptionParams, TensorSerializer, is_vllm_serialized_tensorizer,
13 - load_with_tensorizer, open_stream)
14 +from vllm.model_executor.model_loader.tensorizer import (
15 + EncryptionParams, TensorizerConfig, TensorSerializer,
16 + is_vllm_serialized_tensorizer, load_with_tensorizer, open_stream)
17
18 prompts = [
19 "Hello, my name is",
@@ -22,6 +25,8 @@ prompts = [
25 sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
26
27 model_ref = "facebook/opt-125m"
28 +tensorize_model_for_testing_script = os.path.join(
29 + os.path.dirname(__file__), "tensorize_vllm_model_for_testing.py")
30
31
32 def is_curl_installed():
@@ -38,7 +43,7 @@ def tensorizer_config():
43 return config
44
45
41 -@patch('vllm.model_executor.tensorizer_loader.TensorizerAgent')
46 +@patch('vllm.model_executor.model_loader.tensorizer.TensorizerAgent')
47 def test_load_with_tensorizer(mock_agent, tensorizer_config):
48 mock_linear_method = MagicMock()
49 mock_agent_instance = mock_agent.return_value
@@ -81,11 +86,13 @@ def test_deserialized_vllm_model_has_same_outputs(vllm_runner, tmp_path):
86 del vllm_model, model
87 gc.collect()
88 torch.cuda.empty_cache()
84 - loaded_vllm_model = vllm_runner(model_ref,
85 - load_format="tensorizer",
86 - tensorizer_uri=model_path,
87 - num_readers=1,
88 - vllm_tensorized=True)
89 + loaded_vllm_model = vllm_runner(
90 + model_ref,
91 + load_format="tensorizer",
92 + model_loader_extra_config=TensorizerConfig(tensorizer_uri=model_path,
93 + num_readers=1,
94 + vllm_tensorized=True),
95 + )
96 deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
97
98 # Assumes SamplingParams being seeded ensures the outputs are deterministic
@@ -97,14 +104,14 @@ def test_can_deserialize_s3(vllm_runner):
104 model_ref = "EleutherAI/pythia-1.4b"
105 tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
106
100 - loaded_hf_model = vllm_runner(
101 - model_ref,
102 - tensorizer_uri=tensorized_path,
103 - load_format="tensorizer",
104 - num_readers=1,
105 - vllm_tensorized=False,
106 - s3_endpoint="object.ord1.coreweave.com",
107 - )
107 + loaded_hf_model = vllm_runner(model_ref,
108 + load_format="tensorizer",
109 + model_loader_extra_config=TensorizerConfig(
110 + tensorizer_uri=tensorized_path,
111 + num_readers=1,
112 + vllm_tensorized=False,
113 + s3_endpoint="object.ord1.coreweave.com",
114 + ))
115
116 deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params)
117
@@ -131,11 +138,12 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
138 gc.collect()
139 torch.cuda.empty_cache()
140 loaded_vllm_model = vllm_runner(model_ref,
134 - tensorizer_uri=model_path,
141 load_format="tensorizer",
136 - encryption_keyfile=key_path,
137 - num_readers=1,
138 - vllm_tensorized=True)
142 + model_loader_extra_config=TensorizerConfig(
143 + tensorizer_uri=model_path,
144 + encryption_keyfile=key_path,
145 + num_readers=1,
146 + vllm_tensorized=True))
147
148 deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
149
@@ -156,10 +164,11 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
164 gc.collect()
165 torch.cuda.empty_cache()
166 loaded_hf_model = vllm_runner(model_ref,
159 - tensorizer_uri=model_path,
167 load_format="tensorizer",
161 - num_readers=1,
162 - vllm_tensorized=False)
168 + model_loader_extra_config=TensorizerConfig(
169 + tensorizer_uri=model_path,
170 + num_readers=1,
171 + vllm_tensorized=False))
172
173 deserialized_outputs = loaded_hf_model.generate_greedy(
174 prompts, max_tokens=max_tokens)
@@ -190,10 +199,12 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
199 torch.cuda.empty_cache()
200 loaded_vllm_model = vllm_runner(
201 model_ref,
193 - tensorizer_uri=model_path,
202 load_format="tensorizer",
195 - num_readers=1,
196 - vllm_tensorized=True,
203 + model_loader_extra_config=TensorizerConfig(
204 + tensorizer_uri=model_path,
205 + num_readers=1,
206 + vllm_tensorized=True,
207 + ),
208 enable_lora=True,
209 max_loras=1,
210 max_lora_rank=8,
@@ -208,16 +219,18 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
219
220 def test_load_without_tensorizer_load_format(vllm_runner):
221 with pytest.raises(ValueError):
211 - vllm_runner(model_ref, tensorizer_uri="test")
222 + vllm_runner(model_ref,
223 + model_loader_extra_config=TensorizerConfig(
224 + tensorizer_uri="test", vllm_tensorized=False))
225
226
227 @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
228 def test_tensorize_vllm_model(tmp_path):
229 # Test serialize command
230 serialize_args = [
218 - "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
219 - model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
220 - tmp_path, "--suffix", "tests"
231 + "python3", tensorize_model_for_testing_script, "--model", model_ref,
232 + "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
233 + "--suffix", "tests"
234 ]
235 result = subprocess.run(serialize_args, capture_output=True, text=True)
236 print(result.stdout) # Print the output of the serialize command
@@ -229,8 +242,8 @@ def test_tensorize_vllm_model(tmp_path):
242
243 # Test deserialize command
244 deserialize_args = [
232 - "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
233 - model_ref, "--dtype", "float16", "deserialize", "--path-to-tensors",
245 + "python3", tensorize_model_for_testing_script, "--model", model_ref,
246 + "--dtype", "float16", "deserialize", "--path-to-tensors",
247 path_to_tensors
248 ]
249 result = subprocess.run(deserialize_args, capture_output=True, text=True)
@@ -242,9 +255,9 @@ def test_tensorize_vllm_model(tmp_path):
255 def test_openai_apiserver_with_tensorizer(tmp_path):
256 ## Serialize model
257 serialize_args = [
245 - "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
246 - model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
247 - tmp_path, "--suffix", "tests"
258 + "python3", tensorize_model_for_testing_script, "--model", model_ref,
259 + "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
260 + "--suffix", "tests"
261 ]
262 result = subprocess.run(serialize_args, capture_output=True, text=True)
263 print(result.stdout) # Print the output of the serialize command
@@ -253,25 +266,47 @@ def test_openai_apiserver_with_tensorizer(tmp_path):
266 f"\n{result.stdout}\n{result.stderr}")
267
268 path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
269 + model_loader_extra_config = {
270 + "tensorizer_uri": path_to_tensors,
271 + "vllm_tensorized": True
272 + }
273
274 ## Start OpenAI API server
275 openai_args = [
276 "--model", model_ref, "--dtype", "float16", "--load-format",
260 - "tensorizer", "--tensorizer-uri", path_to_tensors, "--vllm-tensorized",
261 - "--port", "8000"
277 + "tensorizer", "--model-loader-extra-config",
278 + json.dumps(model_loader_extra_config), "--port", "8000"
279 ]
280
281 server = ServerRunner.remote(openai_args)
282
283 + assert ray.get(server.ready.remote())
284 print("Server ready.")
267 - assert server.ready.remote()
285 +
286 + client = openai.OpenAI(
287 + base_url="http://localhost:8000/v1",
288 + api_key="token-abc123",
289 + )
290 + completion = client.completions.create(model=model_ref,
291 + prompt="Hello, my name is",
292 + max_tokens=5,
293 + temperature=0.0)
294 +
295 + assert completion.id is not None
296 + assert completion.choices is not None and len(completion.choices) == 1
297 + assert completion.choices[0].text is not None and len(
298 + completion.choices[0].text) >= 5
299 + assert completion.choices[0].finish_reason == "length"
300 + assert completion.usage == openai.types.CompletionUsage(
301 + completion_tokens=5, prompt_tokens=6, total_tokens=11)
302
303
304 def test_raise_value_error_on_invalid_load_format(vllm_runner):
305 with pytest.raises(ValueError):
306 vllm_runner(model_ref,
307 load_format="safetensors",
274 - tensorizer_uri="test")
308 + model_loader_extra_config=TensorizerConfig(
309 + tensorizer_uri="test", vllm_tensorized=False))
310
311
312 def test_tensorizer_with_tp(vllm_runner):
@@ -281,22 +316,12 @@ def test_tensorizer_with_tp(vllm_runner):
316
317 vllm_runner(
318 model_ref,
284 - tensorizer_uri=tensorized_path,
319 load_format="tensorizer",
286 - num_readers=1,
287 - vllm_tensorized=False,
288 - s3_endpoint="object.ord1.coreweave.com",
320 + model_loader_extra_config=TensorizerConfig(
321 + tensorizer_uri=tensorized_path,
322 + num_readers=1,
323 + vllm_tensorized=False,
324 + s3_endpoint="object.ord1.coreweave.com",
325 + ),
326 tensor_parallel_size=2,
327 )
291 -
292 -
293 -@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
294 -def test_tensorizer_warn_quant(tmp_path):
295 - model_ref = "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit"
296 - serialize_args = [
297 - "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
298 - model_ref, "--quantization", "gptq", "--tensorizer-uri", "test",
299 - "serialize", "--serialized-directory", tmp_path, "--suffix", "tests"
300 - ]
301 - result = subprocess.run(serialize_args, capture_output=True, text=True)
302 - assert 'PerformanceWarning' in result.stderr
tests/test_config.py
-4
@@ -11,8 +11,6 @@ def test_get_sliding_window():
11 "Qwen/Qwen1.5-7B",
12 tokenizer_mode="auto",
13 trust_remote_code=False,
14 - download_dir=None,
15 - load_format="dummy",
14 seed=0,
15 dtype="float16",
16 revision=None,
@@ -30,8 +28,6 @@ def test_get_sliding_window():
28 "mistralai/Mistral-7B-v0.1",
29 tokenizer_mode="auto",
30 trust_remote_code=False,
33 - download_dir=None,
34 - load_format="dummy",
31 seed=0,
32 dtype="float16",
33 revision=None,
tests/test_logits_processor.py
+6 -1
@@ -37,7 +37,12 @@ def _prepare_test(
37 1e-2,
38 dtype=input_tensor.dtype)
39 logits_processor = MockLogitsProcessor(32000, 0.5, fake_logits)
40 - model_runner = ModelRunner(None, None, None, None, None)
40 + model_runner = ModelRunner(model_config=None,
41 + parallel_config=None,
42 + scheduler_config=None,
43 + device_config=None,
44 + load_config=None,
45 + lora_config=None)
46 return input_tensor, fake_logits, logits_processor, model_runner
47
48
tests/worker/test_model_runner.py
+24 -15
@@ -12,7 +12,12 @@ def test_prepare_prompt(batch_size):
12 100000,
13 100000,
14 enable_chunked_prefill=False)
15 - model_runner = ModelRunner(None, None, scheduler_config, None, None)
15 + model_runner = ModelRunner(model_config=None,
16 + parallel_config=None,
17 + scheduler_config=scheduler_config,
18 + device_config=None,
19 + load_config=None,
20 + lora_config=None)
21 model_runner.set_block_size(16)
22
23 prompt_lens = []
@@ -118,8 +123,6 @@ def test_prepare_decode_cuda_graph(batch_size):
123 "facebook/opt-125m",
124 tokenizer_mode="auto",
125 trust_remote_code=False,
121 - download_dir=None,
122 - load_format="dummy",
126 seed=0,
127 dtype="float16",
128 revision=None,
@@ -129,8 +132,12 @@ def test_prepare_decode_cuda_graph(batch_size):
132 100000,
133 100000,
134 enable_chunked_prefill=False)
132 - model_runner = ModelRunner(model_config, None, scheduler_config, None,
133 - None)
135 + model_runner = ModelRunner(model_config=model_config,
136 + parallel_config=None,
137 + scheduler_config=scheduler_config,
138 + device_config=None,
139 + load_config=None,
140 + lora_config=None)
141 model_runner.set_block_size(16)
142
143 prompt_lens = []
@@ -205,14 +212,17 @@ def test_empty_seq_group():
212 "facebook/opt-125m",
213 tokenizer_mode="auto",
214 trust_remote_code=False,
208 - download_dir=None,
209 - load_format="dummy",
215 seed=0,
216 dtype="float16",
217 revision=None,
218 enforce_eager=False,
219 )
215 - model_runner = ModelRunner(model_config, None, None, None, None)
220 + model_runner = ModelRunner(model_config=model_config,
221 + parallel_config=None,
222 + scheduler_config=None,
223 + device_config=None,
224 + load_config=None,
225 + lora_config=None)
226 model_runner.set_block_size(16)
227 seq_group_metadata_list = []
228 input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
@@ -251,8 +261,6 @@ def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
261 "facebook/opt-125m",
262 tokenizer_mode="auto",
263 trust_remote_code=False,
254 - download_dir=None,
255 - load_format="dummy",
264 seed=0,
265 dtype="float16",
266 revision=None,
@@ -262,11 +270,12 @@ def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
270 100000,
271 100000,
272 enable_chunked_prefill=True)
265 - model_runner = ModelRunner(model_config,
266 - None,
267 - scheduler_config,
268 - None,
269 - None,
273 + model_runner = ModelRunner(model_config=model_config,
274 + parallel_config=None,
275 + scheduler_config=scheduler_config,
276 + device_config=None,
277 + load_config=None,
278 + lora_config=None,
279 is_driver_worker=True)
280 model_runner.set_block_size(16)
281
tests/worker/test_swap.py
+1
@@ -23,6 +23,7 @@ def test_swap() -> None:
23 scheduler_config=engine_config.scheduler_config,
24 device_config=engine_config.device_config,
25 cache_config=engine_config.cache_config,
26 + load_config=engine_config.load_config,
27 local_rank=0,
28 rank=0,
29 distributed_init_method=distributed_init_method,
vllm/config.py
+66 -135
@@ -1,9 +1,7 @@
1 import enum
2 -import io
2 import json
3 import os
5 -import typing
6 -from dataclasses import dataclass, fields
4 +from dataclasses import dataclass, field, fields
5 from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
6
7 import torch
@@ -18,10 +16,14 @@ from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
16 if TYPE_CHECKING:
17 from ray.util.placement_group import PlacementGroup
18
21 - from vllm.model_executor.tensorizer_loader import TensorizerArgs
19 + from vllm.model_executor.model_loader.loader import BaseModelLoader
20
21 logger = init_logger(__name__)
22
23 +# If true, will load models from ModelScope instead of Hugging Face Hub.
24 +VLLM_USE_MODELSCOPE = os.environ.get("VLLM_USE_MODELSCOPE",
25 + "False").lower() == "true"
26 +
27 _GB = 1 << 30
28
29
@@ -35,18 +37,6 @@ class ModelConfig:
37 available, and "slow" will always use the slow tokenizer.
38 trust_remote_code: Trust remote code (e.g., from HuggingFace) when
39 downloading the model and tokenizer.
38 - download_dir: Directory to download and load the weights, default to the
39 - default cache directory of huggingface.
40 - load_format: The format of the model weights to load:
41 - "auto" will try to load the weights in the safetensors format and
42 - fall back to the pytorch bin format if safetensors format is
43 - not available.
44 - "pt" will load the weights in the pytorch bin format.
45 - "safetensors" will load the weights in the safetensors format.
46 - "npcache" will load the weights in pytorch format and store
47 - a numpy cache to speed up the loading.
48 - "dummy" will initialize the weights with random values, which is
49 - mainly for profiling.
40 dtype: Data type for model weights and activations. The "auto" option
41 will use FP16 precision for FP32 and FP16 models, and BF16 precision
42 for BF16 models.
@@ -83,8 +73,6 @@ class ModelConfig:
73 tokenizer: str,
74 tokenizer_mode: str,
75 trust_remote_code: bool,
86 - download_dir: Optional[str],
87 - load_format: str,
76 dtype: Union[str, torch.dtype],
77 seed: int,
78 revision: Optional[str] = None,
@@ -101,8 +89,6 @@ class ModelConfig:
89 self.tokenizer = tokenizer
90 self.tokenizer_mode = tokenizer_mode
91 self.trust_remote_code = trust_remote_code
104 - self.download_dir = download_dir
105 - self.load_format = load_format
92 self.seed = seed
93 self.revision = revision
94 self.code_revision = code_revision
@@ -113,64 +99,16 @@ class ModelConfig:
99 self.max_context_len_to_capture = max_context_len_to_capture
100 self.max_logprobs = max_logprobs
101
116 - if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
117 - # download model from ModelScope hub,
118 - # lazy import so that modelscope is not required for normal use.
119 - # pylint: disable=C.
120 - from modelscope.hub.snapshot_download import snapshot_download
121 -
122 - if not os.path.exists(model):
123 - model_path = snapshot_download(model_id=model,
124 - cache_dir=download_dir,
125 - revision=revision)
126 - else:
127 - model_path = model
128 - self.model = model_path
129 - self.download_dir = model_path
130 - self.tokenizer = model_path
131 -
102 self.hf_config = get_config(self.model, trust_remote_code, revision,
103 code_revision)
104 self.hf_text_config = get_hf_text_config(self.hf_config)
105 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
106 self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
107 max_model_len)
138 - self._verify_load_format()
108 self._verify_tokenizer_mode()
109 self._verify_quantization()
110 self._verify_cuda_graph()
111
143 - def _verify_load_format(self) -> None:
144 - load_format = self.load_format.lower()
145 - supported_load_format = [
146 - "auto", "pt", "safetensors", "npcache", "dummy", "tensorizer"
147 - ]
148 - rocm_not_supported_load_format: List[str] = []
149 - if load_format not in supported_load_format:
150 - raise ValueError(
151 - f"Unknown load format: {self.load_format}. Must be one of "
152 - "'auto', 'pt', 'safetensors', 'npcache', 'tensorizer', or "
153 - "'dummy'.")
154 - if is_hip() and load_format in rocm_not_supported_load_format:
155 - rocm_supported_load_format = [
156 - f for f in supported_load_format
157 - if (f not in rocm_not_supported_load_format)
158 - ]
159 - raise ValueError(
160 - f"load format '{load_format}' is not supported in ROCm. "
161 - f"Supported load format are "
162 - f"{rocm_supported_load_format}")
163 -
164 - # TODO: Remove this check once HF updates the pt weights of Mixtral.
165 - architectures = getattr(self.hf_config, "architectures", [])
166 - # architectures can be None instead of []
167 - if architectures and "MixtralForCausalLM" in architectures \
168 - and load_format == "pt":
169 - raise ValueError(
170 - "Currently, the 'pt' format is not supported for Mixtral. "
171 - "Please use the 'safetensors' format instead. ")
172 - self.load_format = load_format
173 -
112 def _verify_tokenizer_mode(self) -> None:
113 tokenizer_mode = self.tokenizer_mode.lower()
114 if tokenizer_mode not in ["auto", "slow"]:
@@ -471,6 +409,65 @@ class TokenizerPoolConfig:
409 return tokenizer_pool_config
410
411
412 +class LoadFormat(str, enum.Enum):
413 + AUTO = "auto"
414 + PT = "pt"
415 + SAFETENSORS = "safetensors"
416 + NPCACHE = "npcache"
417 + DUMMY = "dummy"
418 + TENSORIZER = "tensorizer"
419 +
420 +
421 +@dataclass
422 +class LoadConfig:
423 + """
424 + download_dir: Directory to download and load the weights, default to the
425 + default cache directory of huggingface.
426 + load_format: The format of the model weights to load:
427 + "auto" will try to load the weights in the safetensors format and
428 + fall back to the pytorch bin format if safetensors format is
429 + not available.
430 + "pt" will load the weights in the pytorch bin format.
431 + "safetensors" will load the weights in the safetensors format.
432 + "npcache" will load the weights in pytorch format and store
433 + a numpy cache to speed up the loading.
434 + "dummy" will initialize the weights with random values, which is
435 + mainly for profiling.
436 + "tensorizer" will use CoreWeave's tensorizer library for
437 + fast weight loading.
438 + """
439 +
440 + load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO
441 + download_dir: Optional[str] = None
442 + model_loader_extra_config: Optional[Union[str, dict]] = field(
443 + default_factory=dict)
444 +
445 + def __post_init__(self):
446 + model_loader_extra_config = self.model_loader_extra_config or {}
447 + if isinstance(model_loader_extra_config, str):
448 + self.model_loader_extra_config = json.loads(
449 + model_loader_extra_config)
450 + self._verify_load_format()
451 +
452 + def _verify_load_format(self) -> None:
453 + if not isinstance(self.load_format, str):
454 + return
455 +
456 + load_format = self.load_format.lower()
457 + self.load_format = LoadFormat(load_format)
458 +
459 + rocm_not_supported_load_format: List[str] = []
460 + if is_hip() and load_format in rocm_not_supported_load_format:
461 + rocm_supported_load_format = [
462 + f for f in LoadFormat.__members__
463 + if (f not in rocm_not_supported_load_format)
464 + ]
465 + raise ValueError(
466 + f"load format '{load_format}' is not supported in ROCm. "
467 + f"Supported load formats are "
468 + f"{rocm_supported_load_format}")
469 +
470 +
471 class ParallelConfig:
472 """Configuration for the distributed execution.
473
@@ -699,8 +696,6 @@ class SpeculativeConfig:
696 tokenizer=target_model_config.tokenizer,
697 tokenizer_mode=target_model_config.tokenizer_mode,
698 trust_remote_code=target_model_config.trust_remote_code,
702 - download_dir=target_model_config.download_dir,
703 - load_format=target_model_config.load_format,
699 dtype=target_model_config.dtype,
700 seed=target_model_config.seed,
701 revision=draft_revision,
@@ -887,65 +882,6 @@ class VisionLanguageConfig:
882 f"{[x.name for x in cls.ImageInputType]}.") from e
883
884
890 -@dataclass
891 -class TensorizerConfig:
892 - tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
893 - str, bytes, os.PathLike, int]
894 - vllm_tensorized: bool
895 - verify_hash: Optional[bool] = False
896 - num_readers: Optional[int] = 1
897 - encryption_keyfile: Optional[str] = None
898 - s3_access_key_id: Optional[str] = None
899 - s3_secret_access_key: Optional[str] = None
900 - s3_endpoint: Optional[str] = None
901 - model_class: Optional[torch.nn.Module] = None
902 - hf_config: Optional[PretrainedConfig] = None
903 - dtype: Union[str, torch.dtype] = None
904 -
905 - def _construct_tensorizer_args(self) -> "TensorizerArgs":
906 - from vllm.model_executor.tensorizer_loader import TensorizerArgs
907 - tensorizer_args = {
908 - "tensorizer_uri": self.tensorizer_uri,
909 - "vllm_tensorized": self.vllm_tensorized,
910 - "verify_hash": self.verify_hash,
911 - "num_readers": self.num_readers,
912 - "encryption_keyfile": self.encryption_keyfile,
913 - "s3_access_key_id": self.s3_access_key_id,
914 - "s3_secret_access_key": self.s3_secret_access_key,
915 - "s3_endpoint": self.s3_endpoint,
916 - }
917 - return TensorizerArgs(**tensorizer_args)
918 -
919 - def verify_with_parallel_config(
920 - self,
921 - parallel_config: "ParallelConfig",
922 - ) -> None:
923 - if (parallel_config.tensor_parallel_size > 1
924 - and self.tensorizer_uri is not None):
925 - raise ValueError(
926 - "Loading to multiple GPUs is not currently supported with "
927 - "vLLM-serialized models. Please set tensor_parallel_size=1."
928 - " or use a non-vLLM-serialized model, such as a "
929 - "serialized Hugging Face `PretrainedModel`.")
930 -
931 - def verify_with_model_config(self, model_config) -> None:
932 - if (model_config.quantization is not None
933 - and self.tensorizer_uri is not None):
934 - from vllm.model_executor.tensorizer_loader import (
935 - tensorizer_warning)
936 - tensorizer_warning(
937 - "Loading a model using Tensorizer with quantization on vLLM"
938 - " is unstable and may lead to errors.")
939 -
940 - if (model_config.load_format != "tensorizer"
941 - and self.tensorizer_uri is not None):
942 - raise ValueError(
943 - "A tensorizer uri was passed for tensorizer loading, but the "
944 - f"load format was set to {model_config.load_format}. "
945 - "Please set the load format to 'tensorizer' to use "
946 - f"tensorizer args.")
947 -
948 -
885 _STR_DTYPE_TO_TORCH_DTYPE = {
886 "half": torch.float16,
887 "float16": torch.float16,
@@ -1105,11 +1041,11 @@ class EngineConfig:
1041 parallel_config: ParallelConfig
1042 scheduler_config: SchedulerConfig
1043 device_config: DeviceConfig
1044 + load_config: LoadConfig
1045 lora_config: Optional[LoRAConfig]
1046 vision_language_config: Optional[VisionLanguageConfig]
1047 speculative_config: Optional[SpeculativeConfig]
1048 decoding_config: Optional[DecodingConfig]
1112 - tensorizer_config: Optional[TensorizerConfig]
1049
1050 def __post_init__(self):
1051 """Verify configs are valid & consistent with each other.
@@ -1117,11 +1053,6 @@ class EngineConfig:
1053 self.model_config.verify_with_parallel_config(self.parallel_config)
1054 self.cache_config.verify_with_parallel_config(self.parallel_config)
1055
1120 - if self.tensorizer_config:
1121 - self.tensorizer_config.verify_with_parallel_config(
1122 - self.parallel_config)
1123 - self.tensorizer_config.verify_with_model_config(self.model_config)
1124 -
1056 if self.lora_config:
1057 self.lora_config.verify_with_model_config(self.model_config)
1058 self.lora_config.verify_with_scheduler_config(
vllm/engine/arg_utils.py
+25 -34
@@ -1,15 +1,12 @@
1 import argparse
2 import dataclasses
3 -import io
4 -import os
3 from dataclasses import dataclass
6 -from typing import BinaryIO, Optional, Union
4 +from typing import Optional
5
6 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9 - EngineConfig, LoRAConfig, ModelConfig, ParallelConfig,
10 - SchedulerConfig, SpeculativeConfig, TensorizerConfig,
7 + EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
8 + ParallelConfig, SchedulerConfig, SpeculativeConfig,
9 TokenizerPoolConfig, VisionLanguageConfig)
12 -from vllm.model_executor.tensorizer_loader import TensorizerArgs
10 from vllm.utils import str_to_int_tuple
11
12
@@ -60,17 +57,7 @@ class EngineArgs:
57 ray_workers_use_nsight: bool = False
58 num_gpu_blocks_override: Optional[int] = None
59 num_lookahead_slots: int = 0
63 -
64 - # Tensorizer configuration parameters
65 - tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, BinaryIO, str,
66 - bytes, os.PathLike, int] = None
67 - vllm_tensorized: bool = False
68 - verify_hash: Optional[bool] = False
69 - num_readers: Optional[int] = 1
70 - encryption_keyfile: Optional[str] = None
71 - s3_access_key_id: Optional[str] = None
72 - s3_secret_access_key: Optional[str] = None
73 - s3_endpoint: Optional[str] = None
60 + model_loader_extra_config: Optional[dict] = None
61
62 # Related to Vision-language models such as llava
63 image_input_type: Optional[str] = None
@@ -429,7 +416,16 @@ class EngineArgs:
416 default=None,
417 help='The number of speculative tokens to sample from '
418 'the draft model in speculative decoding')
432 - parser = TensorizerArgs.add_cli_args(parser)
419 +
420 + parser.add_argument('--model-loader-extra-config',
421 + type=str,
422 + default=EngineArgs.model_loader_extra_config,
423 + help='Extra config for model loader. '
424 + 'This will be passed to the model loader '
425 + 'corresponding to the chosen load_format. '
426 + 'This should be a JSON string that will be '
427 + 'parsed into a dictionary.')
428 +
429 return parser
430
431 @classmethod
@@ -444,11 +440,11 @@ class EngineArgs:
440 device_config = DeviceConfig(self.device)
441 model_config = ModelConfig(
442 self.model, self.tokenizer, self.tokenizer_mode,
447 - self.trust_remote_code, self.download_dir, self.load_format,
448 - self.dtype, self.seed, self.revision, self.code_revision,
449 - self.tokenizer_revision, self.max_model_len, self.quantization,
450 - self.quantization_param_path, self.enforce_eager,
451 - self.max_context_len_to_capture, self.max_logprobs)
443 + self.trust_remote_code, self.dtype, self.seed, self.revision,
444 + self.code_revision, self.tokenizer_revision, self.max_model_len,
445 + self.quantization, self.quantization_param_path,
446 + self.enforce_eager, self.max_context_len_to_capture,
447 + self.max_logprobs)
448 cache_config = CacheConfig(self.block_size,
449 self.gpu_memory_utilization,
450 self.swap_space, self.kv_cache_dtype,
@@ -492,15 +488,10 @@ class EngineArgs:
488 max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
489 and self.max_cpu_loras > 0 else None) if self.enable_lora else None
490
495 - tensorizer_config = TensorizerConfig(
496 - tensorizer_uri=self.tensorizer_uri,
497 - vllm_tensorized=self.vllm_tensorized,
498 - verify_hash=self.verify_hash,
499 - num_readers=self.num_readers,
500 - encryption_keyfile=self.encryption_keyfile,
501 - s3_access_key_id=self.s3_access_key_id,
502 - s3_secret_access_key=self.s3_secret_access_key,
503 - s3_endpoint=self.s3_endpoint,
491 + load_config = LoadConfig(
492 + load_format=self.load_format,
493 + download_dir=self.download_dir,
494 + model_loader_extra_config=self.model_loader_extra_config,
495 )
496
497 if self.image_input_type:
@@ -530,8 +521,8 @@ class EngineArgs:
521 lora_config=lora_config,
522 vision_language_config=vision_language_config,
523 speculative_config=speculative_config,
533 - decoding_config=decoding_config,
534 - tensorizer_config=tensorizer_config)
524 + load_config=load_config,
525 + decoding_config=decoding_config)
526
527
528 @dataclass
vllm/engine/llm_engine.py
+8 -11
@@ -4,9 +4,9 @@ from typing import Iterable, List, Optional, Tuple, Type, Union
4 from transformers import PreTrainedTokenizer
5
6 import vllm
7 -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoRAConfig,
8 - ModelConfig, ParallelConfig, SchedulerConfig,
9 - SpeculativeConfig, TensorizerConfig,
7 +from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
8 + LoRAConfig, ModelConfig, ParallelConfig,
9 + SchedulerConfig, SpeculativeConfig,
10 VisionLanguageConfig)
11 from vllm.core.scheduler import Scheduler, SchedulerOutputs
12 from vllm.engine.arg_utils import EngineArgs
@@ -72,11 +72,11 @@ class LLMEngine:
72 parallel_config: ParallelConfig,
73 scheduler_config: SchedulerConfig,
74 device_config: DeviceConfig,
75 + load_config: LoadConfig,
76 lora_config: Optional[LoRAConfig],
77 vision_language_config: Optional[VisionLanguageConfig],
78 speculative_config: Optional[SpeculativeConfig],
79 decoding_config: Optional[DecodingConfig],
79 - tensorizer_config: Optional[TensorizerConfig],
80 executor_class: Type[ExecutorBase],
81 log_stats: bool,
82 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -92,8 +92,8 @@ class LLMEngine:
92 f"trust_remote_code={model_config.trust_remote_code}, "
93 f"dtype={model_config.dtype}, "
94 f"max_seq_len={model_config.max_model_len}, "
95 - f"download_dir={model_config.download_dir!r}, "
96 - f"load_format={model_config.load_format}, "
95 + f"download_dir={load_config.download_dir!r}, "
96 + f"load_format={load_config.load_format}, "
97 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
98 f"disable_custom_all_reduce="
99 f"{parallel_config.disable_custom_all_reduce}, "
@@ -114,8 +114,8 @@ class LLMEngine:
114 self.scheduler_config = scheduler_config
115 self.device_config = device_config
116 self.speculative_config = speculative_config
117 + self.load_config = load_config
118 self.decoding_config = decoding_config or DecodingConfig()
118 - self.tensorizer_config = tensorizer_config
119 self.log_stats = log_stats
120
121 self._init_tokenizer()
@@ -131,7 +131,7 @@ class LLMEngine:
131 lora_config=lora_config,
132 vision_language_config=vision_language_config,
133 speculative_config=speculative_config,
134 - tensorizer_config=tensorizer_config,
134 + load_config=load_config,
135 )
136
137 self._initialize_kv_caches()
@@ -271,9 +271,6 @@ class LLMEngine:
271 def _verify_args(self) -> None:
272 self.model_config.verify_with_parallel_config(self.parallel_config)
273 self.cache_config.verify_with_parallel_config(self.parallel_config)
274 - if self.tensorizer_config:
275 - self.tensorizer_config.verify_with_parallel_config(
276 - self.parallel_config)
274 if self.lora_config:
275 self.lora_config.verify_with_model_config(self.model_config)
276 self.lora_config.verify_with_scheduler_config(
vllm/executor/cpu_executor.py
+1
@@ -40,6 +40,7 @@ class CPUExecutor(ExecutorBase):
40 scheduler_config=self.scheduler_config,
41 device_config=self.device_config,
42 cache_config=self.cache_config,
43 + load_config=self.load_config,
44 local_rank=0,
45 rank=0,
46 distributed_init_method=distributed_init_method,
vllm/executor/executor_base.py
+5 -5
@@ -1,9 +1,9 @@
1 from abc import ABC, abstractmethod
2 from typing import Dict, List, Optional, Set, Tuple
3
4 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5 - ParallelConfig, SchedulerConfig, SpeculativeConfig,
6 - TensorizerConfig, VisionLanguageConfig)
4 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5 + ModelConfig, ParallelConfig, SchedulerConfig,
6 + SpeculativeConfig, VisionLanguageConfig)
7 from vllm.lora.request import LoRARequest
8 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
@@ -23,20 +23,20 @@ class ExecutorBase(ABC):
23 parallel_config: ParallelConfig,
24 scheduler_config: SchedulerConfig,
25 device_config: DeviceConfig,
26 + load_config: LoadConfig,
27 lora_config: Optional[LoRAConfig],
28 vision_language_config: Optional[VisionLanguageConfig],
29 speculative_config: Optional[SpeculativeConfig],
29 - tensorizer_config: Optional[TensorizerConfig],
30 ) -> None:
31 self.model_config = model_config
32 self.cache_config = cache_config
33 self.lora_config = lora_config
34 + self.load_config = load_config
35 self.parallel_config = parallel_config
36 self.scheduler_config = scheduler_config
37 self.device_config = device_config
38 self.vision_language_config = vision_language_config
39 self.speculative_config = speculative_config
39 - self.tensorizer_config = tensorizer_config
40
41 self._init_executor()
42
vllm/executor/gpu_executor.py
+1 -1
@@ -35,12 +35,12 @@ class GPUExecutor(ExecutorBase):
35 scheduler_config=self.scheduler_config,
36 device_config=self.device_config,
37 cache_config=self.cache_config,
38 + load_config=self.load_config,
39 local_rank=0,
40 rank=0,
41 distributed_init_method=distributed_init_method,
42 lora_config=self.lora_config,
43 vision_language_config=self.vision_language_config,
43 - tensorizer_config=self.tensorizer_config,
44 is_driver_worker=True,
45 )
46 self.driver_worker.init_device()
vllm/executor/ray_gpu_executor.py
+3 -2
@@ -147,6 +147,7 @@ class RayGPUExecutor(ExecutorBase):
147 model_config = copy.deepcopy(self.model_config)
148 parallel_config = copy.deepcopy(self.parallel_config)
149 scheduler_config = copy.deepcopy(self.scheduler_config)
150 + load_config = copy.deepcopy(self.load_config)
151 device_config = copy.deepcopy(self.device_config)
152 lora_config = copy.deepcopy(self.lora_config)
153 cache_config = copy.deepcopy(self.cache_config)
@@ -165,12 +166,12 @@ class RayGPUExecutor(ExecutorBase):
166 scheduler_config=scheduler_config,
167 device_config=device_config,
168 cache_config=cache_config,
169 + load_config=load_config,
170 local_rank=local_rank,
171 rank=rank,
172 distributed_init_method=distributed_init_method,
173 lora_config=lora_config,
174 vision_language_config=vision_language_config,
173 - tensorizer_config=self.tensorizer_config,
175 ))
176
177 # Initialize the driver worker with the Worker class.
@@ -187,7 +188,7 @@ class RayGPUExecutor(ExecutorBase):
188 distributed_init_method=distributed_init_method,
189 lora_config=self.lora_config,
190 vision_language_config=self.vision_language_config,
190 - tensorizer_config=self.tensorizer_config,
191 + load_config=self.load_config,
192 is_driver_worker=True,
193 )
194
vllm/model_executor/model_loader.py deleted
-128
@@ -1,128 +0,0 @@
1 -"""Utilities for selecting and loading models."""
2 -import contextlib
3 -from typing import Tuple, Type
4 -
5 -import torch
6 -from torch import nn
7 -
8 -from vllm.config import DeviceConfig, ModelConfig
9 -from vllm.model_executor.models import ModelRegistry
10 -from vllm.model_executor.models.llava import LlavaForConditionalGeneration
11 -from vllm.model_executor.tensorizer_loader import (
12 - ParameterizedLoadFormat, is_vllm_serialized_tensorizer,
13 - load_with_tensorizer)
14 -from vllm.model_executor.weight_utils import (get_quant_config,
15 - initialize_dummy_weights)
16 -
17 -_VISION_MODEL_CLASSES = [
18 - LlavaForConditionalGeneration,
19 -]
20 -
21 -
22 -@contextlib.contextmanager
23 -def _set_default_torch_dtype(dtype: torch.dtype):
24 - """Sets the default torch dtype to the given dtype."""
25 - old_dtype = torch.get_default_dtype()
26 - torch.set_default_dtype(dtype)
27 - yield
28 - torch.set_default_dtype(old_dtype)
29 -
30 -
31 -def _get_model_architecture(
32 - model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
33 - architectures = getattr(model_config.hf_config, "architectures", [])
34 - # Special handling for quantized Mixtral.
35 - # FIXME(woosuk): This is a temporary hack.
36 - if (model_config.quantization is not None
37 - and "MixtralForCausalLM" in architectures):
38 - architectures = ["QuantMixtralForCausalLM"]
39 -
40 - for arch in architectures:
41 - model_cls = ModelRegistry.load_model_cls(arch)
42 - if model_cls is not None:
43 - return (model_cls, arch)
44 - raise ValueError(
45 - f"Model architectures {architectures} are not supported for now. "
46 - f"Supported architectures: {ModelRegistry.get_supported_archs()}")
47 -
48 -
49 -def get_architecture_class_name(model_config: ModelConfig) -> str:
50 - return _get_model_architecture(model_config)[1]
51 -
52 -
53 -def get_model(model_config: ModelConfig, device_config: DeviceConfig,
54 - **kwargs) -> nn.Module:
55 - lora_config = kwargs.get("lora_config", None)
56 - vision_language_config = kwargs.get("vision_language_config", None)
57 - tensorizer_config = kwargs.get("tensorizer_config", None)
58 - model_class = _get_model_architecture(model_config)[0]
59 -
60 - # Get the (maybe quantized) linear method.
61 - linear_method = None
62 - if model_config.quantization is not None:
63 - quant_config = get_quant_config(model_config)
64 - capability = torch.cuda.get_device_capability()
65 - capability = capability[0] * 10 + capability[1]
66 - if capability < quant_config.get_min_capability():
67 - raise ValueError(
68 - f"The quantization method {model_config.quantization} is not "
69 - "supported for the current GPU. "
70 - f"Minimum capability: {quant_config.get_min_capability()}. "
71 - f"Current capability: {capability}.")
72 - supported_dtypes = quant_config.get_supported_act_dtypes()
73 - if model_config.dtype not in supported_dtypes:
74 - raise ValueError(
75 - f"{model_config.dtype} is not supported for quantization "
76 - f"method {model_config.quantization}. Supported dtypes: "
77 - f"{supported_dtypes}")
78 -
79 - linear_method = quant_config.get_linear_method()
80 -
81 - with _set_default_torch_dtype(model_config.dtype):
82 - # Create a model instance.
83 - # The weights will be initialized as empty tensors.
84 - extra_kwargs = {}
85 - if hasattr(model_class, "supported_lora_modules"):
86 - extra_kwargs["lora_config"] = lora_config
87 - elif lora_config:
88 - raise ValueError(
89 - f"Model {model_class.__name__} does not support LoRA, "
90 - "but LoRA is enabled. Support for this model may "
91 - "be added in the future. If this is important to you, "
92 - "please open an issue on github.")
93 - elif model_class in _VISION_MODEL_CLASSES:
94 - extra_kwargs["vision_language_config"] = vision_language_config
95 -
96 - with torch.device(device_config.device):
97 - if (model_config.load_format == "tensorizer"
98 - and is_vllm_serialized_tensorizer(tensorizer_config)):
99 - extra_kwargs["linear_method"] = linear_method
100 - tensorizer_config.model_class = model_class
101 - tensorizer_config.hf_config = model_config.hf_config
102 - tensorizer_config.dtype = model_config.dtype
103 - model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
104 - return model.eval()
105 - model = model_class(config=model_config.hf_config,
106 - linear_method=linear_method,
107 - **extra_kwargs)
108 - if model_config.load_format == "dummy":
109 - # NOTE(woosuk): For accurate performance evaluation, we assign
110 - # random values to the weights.
111 - initialize_dummy_weights(model)
112 - else:
113 - # Load the weights from the cached or downloaded files.
114 - if model_config.load_format == "tensorizer":
115 - # Provide a dynamic load format for `model.load_weights`
116 - # to retain tensorizer args from CLI.
117 - model_config.load_format = ParameterizedLoadFormat(
118 - model_config.load_format)
119 - model_config.load_format.params = (
120 - tensorizer_config._construct_tensorizer_args())
121 -
122 - model.load_weights(
123 - model_config.model,
124 - model_config.download_dir,
125 - model_config.load_format,
126 - model_config.revision,
127 - )
128 - return model.eval()
vllm/model_executor/model_loader/__init__.py new
+30
@@ -0,0 +1,30 @@
1 +from typing import Optional
2 +
3 +from torch import nn
4 +
5 +from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
6 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
7 +from vllm.model_executor.model_loader.loader import (BaseModelLoader,
8 + get_model_loader)
9 +from vllm.model_executor.model_loader.utils import (
10 + get_architecture_class_name, get_model_architecture)
11 +
12 +
13 +def get_model(
14 + *, model_config: ModelConfig, load_config: LoadConfig,
15 + device_config: DeviceConfig, parallel_config: ParallelConfig,
16 + scheduler_config: SchedulerConfig, lora_config: Optional[LoRAConfig],
17 + vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
18 + loader = get_model_loader(load_config)
19 + return loader.load_model(model_config=model_config,
20 + device_config=device_config,
21 + lora_config=lora_config,
22 + vision_language_config=vision_language_config,
23 + parallel_config=parallel_config,
24 + scheduler_config=scheduler_config)
25 +
26 +
27 +__all__ = [
28 + "get_model", "get_model_loader", "BaseModelLoader",
29 + "get_architecture_class_name", "get_model_architecture"
30 +]
vllm/model_executor/model_loader/loader.py new
+354
@@ -0,0 +1,354 @@
1 +# ruff: noqa: SIM117
2 +import copy
3 +import glob
4 +import os
5 +from abc import ABC, abstractmethod
6 +from typing import (TYPE_CHECKING, Any, Dict, Generator, List, Optional, Tuple,
7 + Type)
8 +
9 +import torch
10 +from torch import nn
11 +
12 +from vllm.config import (VLLM_USE_MODELSCOPE, DeviceConfig, LoadConfig,
13 + LoadFormat, LoRAConfig, ModelConfig, ParallelConfig,
14 + SchedulerConfig, VisionLanguageConfig)
15 +from vllm.logger import init_logger
16 +from vllm.model_executor.model_loader.tensorizer import (
17 + TensorizerConfig, is_vllm_serialized_tensorizer, load_with_tensorizer,
18 + tensorizer_weights_iterator)
19 +from vllm.model_executor.model_loader.utils import (get_model_architecture,
20 + set_default_torch_dtype)
21 +from vllm.model_executor.model_loader.weight_utils import (
22 + download_weights_from_hf, filter_files_not_needed_for_inference,
23 + get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
24 + pt_weights_iterator, safetensors_weights_iterator)
25 +from vllm.model_executor.models.llava import LlavaForConditionalGeneration
26 +
27 +if TYPE_CHECKING:
28 + from vllm.model_executor.layers.linear import LinearMethodBase
29 +
30 +_VISION_MODEL_CLASSES = [
31 + LlavaForConditionalGeneration,
32 +]
33 +
34 +logger = init_logger(__name__)
35 +
36 +
37 +def _get_linear_method(
38 + model_config: ModelConfig,
39 + load_config: LoadConfig) -> Optional["LinearMethodBase"]:
40 + """Get the (maybe quantized) linear method."""
41 + linear_method = None
42 + if model_config.quantization is not None:
43 + quant_config = get_quant_config(model_config, load_config)
44 + capability = torch.cuda.get_device_capability()
45 + capability = capability[0] * 10 + capability[1]
46 + if capability < quant_config.get_min_capability():
47 + raise ValueError(
48 + f"The quantization method {model_config.quantization} is not "
49 + "supported for the current GPU. "
50 + f"Minimum capability: {quant_config.get_min_capability()}. "
51 + f"Current capability: {capability}.")
52 + supported_dtypes = quant_config.get_supported_act_dtypes()
53 + if model_config.dtype not in supported_dtypes:
54 + raise ValueError(
55 + f"{model_config.dtype} is not supported for quantization "
56 + f"method {model_config.quantization}. Supported dtypes: "
57 + f"{supported_dtypes}")
58 +
59 + linear_method = quant_config.get_linear_method()
60 + return linear_method
61 +
62 +
63 +def _get_model_initialization_kwargs(
64 + model_class: Type[nn.Module], lora_config: Optional[LoRAConfig],
65 + vision_language_config: Optional[VisionLanguageConfig]
66 +) -> Dict[str, Any]:
67 + """Get extra kwargs for model initialization."""
68 + extra_kwargs = {}
69 + if hasattr(model_class, "supported_lora_modules"):
70 + extra_kwargs["lora_config"] = lora_config
71 + elif lora_config:
72 + raise ValueError(
73 + f"Model {model_class.__name__} does not support LoRA, "
74 + "but LoRA is enabled. Support for this model may "
75 + "be added in the future. If this is important to you, "
76 + "please open an issue on github.")
77 + elif model_class in _VISION_MODEL_CLASSES:
78 + extra_kwargs["vision_language_config"] = vision_language_config
79 + return extra_kwargs
80 +
81 +
82 +def _initialize_model(
83 + model_config: ModelConfig, load_config: LoadConfig,
84 + lora_config: Optional[LoRAConfig],
85 + vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
86 + """Initialize a model with the given configurations."""
87 + model_class = get_model_architecture(model_config)[0]
88 + linear_method = _get_linear_method(model_config, load_config)
89 +
90 + return model_class(config=model_config.hf_config,
91 + linear_method=linear_method,
92 + **_get_model_initialization_kwargs(
93 + model_class, lora_config, vision_language_config))
94 +
95 +
96 +class BaseModelLoader(ABC):
97 + """Base class for model loaders."""
98 +
99 + def __init__(self, load_config: LoadConfig):
100 + self.load_config = load_config
101 +
102 + @abstractmethod
103 + def load_model(self, *, model_config: ModelConfig,
104 + device_config: DeviceConfig,
105 + lora_config: Optional[LoRAConfig],
106 + vision_language_config: Optional[VisionLanguageConfig],
107 + parallel_config: ParallelConfig,
108 + scheduler_config: SchedulerConfig) -> nn.Module:
109 + """Load a model with the given configurations."""
110 + ...
111 +
112 +
113 +class DefaultModelLoader(BaseModelLoader):
114 + """Model loader that can load different file types from disk."""
115 +
116 + def __init__(self, load_config: LoadConfig):
117 + super().__init__(load_config)
118 + if load_config.model_loader_extra_config:
119 + raise ValueError(f"Model loader extra config is not supported for "
120 + f"load format {load_config.load_format}")
121 +
122 + def _maybe_download_from_modelscope(
123 + self, model: str, revision: Optional[str]) -> Optional[str]:
124 + """Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
125 +
126 + Returns the path to the downloaded model, or None if the model is not
127 + downloaded from ModelScope."""
128 + if VLLM_USE_MODELSCOPE:
129 + # download model from ModelScope hub,
130 + # lazy import so that modelscope is not required for normal use.
131 + # pylint: disable=C.
132 + from modelscope.hub.snapshot_download import snapshot_download
133 +
134 + if not os.path.exists(model):
135 + model_path = snapshot_download(
136 + model_id=model,
137 + cache_dir=self.load_config.download_dir,
138 + revision=revision)
139 + else:
140 + model_path = model
141 + return model_path
142 + return None
143 +
144 + def _prepare_weights(self, model_name_or_path: str,
145 + revision: Optional[str],
146 + fall_back_to_pt: bool) -> Tuple[str, List[str], bool]:
147 + """Prepare weights for the model.
148 +
149 + If the model is not local, it will be downloaded."""
150 + model_name_or_path = self._maybe_download_from_modelscope(
151 + model_name_or_path, revision) or model_name_or_path
152 +
153 + is_local = os.path.isdir(model_name_or_path)
154 + load_format = self.load_config.load_format
155 + use_safetensors = False
156 + # Some quantized models use .pt files for storing the weights.
157 + if load_format == LoadFormat.AUTO:
158 + allow_patterns = ["*.safetensors", "*.bin"]
159 + elif load_format == LoadFormat.SAFETENSORS:
160 + use_safetensors = True
161 + allow_patterns = ["*.safetensors"]
162 + elif load_format == LoadFormat.PT:
163 + allow_patterns = ["*.pt"]
164 + elif load_format == LoadFormat.NPCACHE:
165 + allow_patterns = ["*.bin"]
166 + else:
167 + raise ValueError(f"Unknown load_format: {load_format}")
168 +
169 + if fall_back_to_pt:
170 + allow_patterns += ["*.pt"]
171 +
172 + if not is_local:
173 + hf_folder = download_weights_from_hf(model_name_or_path,
174 + self.load_config.download_dir,
175 + allow_patterns)
176 + else:
177 + hf_folder = model_name_or_path
178 +
179 + hf_weights_files: List[str] = []
180 + for pattern in allow_patterns:
181 + hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
182 + if len(hf_weights_files) > 0:
183 + if pattern == "*.safetensors":
184 + use_safetensors = True
185 + break
186 +
187 + if not use_safetensors:
188 + hf_weights_files = filter_files_not_needed_for_inference(
189 + hf_weights_files)
190 +
191 + if len(hf_weights_files) == 0:
192 + raise RuntimeError(
193 + f"Cannot find any model weights with `{model_name_or_path}`")
194 +
195 + return hf_folder, hf_weights_files, use_safetensors
196 +
197 + def _get_weights_iterator(
198 + self, model_name_or_path: str, revision: Optional[str],
199 + fall_back_to_pt: bool
200 + ) -> Generator[Tuple[str, torch.Tensor], None, None]:
201 + """Get an iterator for the model weights based on the load format."""
202 + hf_folder, hf_weights_files, use_safetensors = self._prepare_weights(
203 + model_name_or_path, revision, fall_back_to_pt)
204 + if self.load_config.load_format == LoadFormat.NPCACHE:
205 + # Currently np_cache only support *.bin checkpoints
206 + assert use_safetensors is False
207 + return np_cache_weights_iterator(model_name_or_path,
208 + self.load_config.download_dir,
209 + hf_folder, hf_weights_files)
210 + if use_safetensors:
211 + return safetensors_weights_iterator(hf_weights_files)
212 + return pt_weights_iterator(hf_weights_files)
213 +
214 + def load_model(self, *, model_config: ModelConfig,
215 + device_config: DeviceConfig,
216 + lora_config: Optional[LoRAConfig],
217 + vision_language_config: Optional[VisionLanguageConfig],
218 + parallel_config: ParallelConfig,
219 + scheduler_config: SchedulerConfig) -> nn.Module:
220 + with set_default_torch_dtype(model_config.dtype):
221 + with torch.device(device_config.device):
222 + model = _initialize_model(model_config, self.load_config,
223 + lora_config, vision_language_config)
224 + model.load_weights(
225 + self._get_weights_iterator(model_config.model,
226 + model_config.revision,
227 + fall_back_to_pt=getattr(
228 + model,
229 + "fall_back_to_pt_during_load",
230 + True)), )
231 + return model.eval()
232 +
233 +
234 +class DummyModelLoader(BaseModelLoader):
235 + """Model loader that will set model weights to random values."""
236 +
237 + def __init__(self, load_config: LoadConfig):
238 + super().__init__(load_config)
239 + if load_config.model_loader_extra_config:
240 + raise ValueError(f"Model loader extra config is not supported for "
241 + f"load format {load_config.load_format}")
242 +
243 + def load_model(self, *, model_config: ModelConfig,
244 + device_config: DeviceConfig,
245 + lora_config: Optional[LoRAConfig],
246 + vision_language_config: Optional[VisionLanguageConfig],
247 + parallel_config: ParallelConfig,
248 + scheduler_config: SchedulerConfig) -> nn.Module:
249 + with set_default_torch_dtype(model_config.dtype):
250 + with torch.device(device_config.device):
251 + model = _initialize_model(model_config, self.load_config,
252 + lora_config, vision_language_config)
253 + # NOTE(woosuk): For accurate performance evaluation, we assign
254 + # random values to the weights.
255 + initialize_dummy_weights(model)
256 + return model.eval()
257 +
258 +
259 +class TensorizerLoader(BaseModelLoader):
260 + """Model loader using CoreWeave's tensorizer library."""
261 +
262 + def __init__(self, load_config: LoadConfig):
263 + super().__init__(load_config)
264 + if isinstance(load_config.model_loader_extra_config, TensorizerConfig):
265 + self.tensorizer_config = load_config.model_loader_extra_config
266 + else:
267 + self.tensorizer_config = TensorizerConfig(
268 + **load_config.model_loader_extra_config)
269 +
270 + def _verify_config(self, model_config: ModelConfig,
271 + parallel_config: ParallelConfig):
272 + self.tensorizer_config.verify_with_model_config(model_config)
273 + self.tensorizer_config.verify_with_parallel_config(parallel_config)
274 +
275 + def _get_weights_iterator(
276 + self) -> Generator[Tuple[str, torch.Tensor], None, None]:
277 + tensorizer_args = self.tensorizer_config._construct_tensorizer_args()
278 + return tensorizer_weights_iterator(tensorizer_args)
279 +
280 + def _load_model_unserialized(
281 + self, model_config: ModelConfig, device_config: DeviceConfig,
282 + lora_config: Optional[LoRAConfig],
283 + vision_language_config: Optional[VisionLanguageConfig]
284 + ) -> nn.Module:
285 + """Load an unserialized model with tensorizer.
286 +
287 + Unserialized here means "not serialized with tensorizer". This
288 + should still be faster than default HuggingFace loading, but will
289 + be slower than loading a tensorizer-serialized model.
290 + """
291 + with set_default_torch_dtype(model_config.dtype):
292 + with torch.device(device_config.device):
293 + model = _initialize_model(model_config, self.load_config,
294 + lora_config, vision_language_config)
295 +
296 + model.load_weights(self._get_weights_iterator())
297 + return model.eval()
298 +
299 + def _load_model_serialized(
300 + self, model_config: ModelConfig, device_config: DeviceConfig,
301 + lora_config: Optional[LoRAConfig],
302 + vision_language_config: Optional[VisionLanguageConfig]
303 + ) -> nn.Module:
304 + """Load a serialized model with tensorizer.
305 +
306 + See the examples/tensorize_vllm_model.py example "
307 + script for serializing vLLM models."""
308 + with set_default_torch_dtype(model_config.dtype):
309 + with torch.device(device_config.device):
310 + model_class = get_model_architecture(model_config)[0]
311 + linear_method = _get_linear_method(model_config,
312 + self.load_config)
313 + extra_kwargs = _get_model_initialization_kwargs(
314 + model_class, lora_config, vision_language_config)
315 + extra_kwargs["linear_method"] = linear_method
316 +
317 + tensorizer_config = copy.copy(self.tensorizer_config)
318 + tensorizer_config.model_class = model_class
319 + tensorizer_config.hf_config = model_config.hf_config
320 + tensorizer_config.dtype = model_config.dtype
321 +
322 + model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
323 + return model.eval()
324 +
325 + def load_model(self, *, model_config: ModelConfig,
326 + device_config: DeviceConfig,
327 + lora_config: Optional[LoRAConfig],
328 + vision_language_config: Optional[VisionLanguageConfig],
329 + parallel_config: ParallelConfig,
330 + scheduler_config: SchedulerConfig) -> nn.Module:
331 + self._verify_config(model_config, parallel_config)
332 +
333 + if is_vllm_serialized_tensorizer(self.tensorizer_config):
334 + return self._load_model_serialized(model_config, device_config,
335 + lora_config,
336 + vision_language_config)
337 + return self._load_model_unserialized(model_config, device_config,
338 + lora_config,
339 + vision_language_config)
340 +
341 +
342 +def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
343 + """Get a model loader based on the load format."""
344 +
345 + if isinstance(load_config.load_format, type):
346 + return load_config.load_format(load_config)
347 +
348 + if load_config.load_format == LoadFormat.DUMMY:
349 + return DummyModelLoader(load_config)
350 +
351 + if load_config.load_format == LoadFormat.TENSORIZER:
352 + return TensorizerLoader(load_config)
353 +
354 + return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/neuron.py renamed
vllm/model_executor/model_loader/tensorizer.py renamed
+82 -34
@@ -4,20 +4,20 @@ import io
4 import os
5 import time
6 import typing
7 -import warnings
7 from dataclasses import dataclass
9 -from typing import Optional, Union
8 +from typing import Generator, Optional, Tuple, Type, Union
9
10 import torch
11 from torch import nn
12 +from transformers import PretrainedConfig
13
14 -from vllm.config import TensorizerConfig
14 +from vllm.config import ModelConfig, ParallelConfig
15 from vllm.logger import init_logger
16 from vllm.model_executor.layers.linear import LinearMethodBase
17 from vllm.model_executor.layers.vocab_parallel_embedding import (
18 VocabParallelEmbedding)
19
20 -tensorizer_load_fail = False
20 +tensorizer_load_fail = None
21
22 try:
23 from tensorizer import (DecryptionParams, EncryptionParams,
@@ -25,51 +25,78 @@ try:
25 from tensorizer.stream_io import open_stream
26 from tensorizer.utils import (convert_bytes, get_mem_usage,
27 no_init_or_tensor)
28 -except ImportError:
29 - tensorizer_load_fail = True
28 +except ImportError as e:
29 + tensorizer_load_fail = e
30
31 __all__ = [
32 'EncryptionParams', 'DecryptionParams', 'TensorDeserializer',
33 'TensorSerializer', 'open_stream', 'convert_bytes', 'get_mem_usage',
34 - 'no_init_or_tensor'
34 + 'no_init_or_tensor', 'TensorizerConfig'
35 ]
36
37 logger = init_logger(__name__)
38
39
40 +@dataclass
41 +class TensorizerConfig:
42 + tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
43 + str, bytes, os.PathLike, int]
44 + vllm_tensorized: bool
45 + verify_hash: Optional[bool] = False
46 + num_readers: Optional[int] = 1
47 + encryption_keyfile: Optional[str] = None
48 + s3_access_key_id: Optional[str] = None
49 + s3_secret_access_key: Optional[str] = None
50 + s3_endpoint: Optional[str] = None
51 + model_class: Optional[Type[torch.nn.Module]] = None
52 + hf_config: Optional[PretrainedConfig] = None
53 + dtype: Optional[Union[str, torch.dtype]] = None
54 +
55 + def _construct_tensorizer_args(self) -> "TensorizerArgs":
56 + tensorizer_args = {
57 + "tensorizer_uri": self.tensorizer_uri,
58 + "vllm_tensorized": self.vllm_tensorized,
59 + "verify_hash": self.verify_hash,
60 + "num_readers": self.num_readers,
61 + "encryption_keyfile": self.encryption_keyfile,
62 + "s3_access_key_id": self.s3_access_key_id,
63 + "s3_secret_access_key": self.s3_secret_access_key,
64 + "s3_endpoint": self.s3_endpoint,
65 + }
66 + return TensorizerArgs(**tensorizer_args)
67 +
68 + def verify_with_parallel_config(
69 + self,
70 + parallel_config: "ParallelConfig",
71 + ) -> None:
72 + if (parallel_config.tensor_parallel_size > 1
73 + and self.tensorizer_uri is not None):
74 + raise ValueError(
75 + "Loading to multiple GPUs is not currently supported with "
76 + "vLLM-serialized models. Please set tensor_parallel_size=1."
77 + " or use a non-vLLM-serialized model, such as a "
78 + "serialized Hugging Face `PretrainedModel`.")
79 +
80 + def verify_with_model_config(self, model_config: "ModelConfig") -> None:
81 + if (model_config.quantization is not None
82 + and self.tensorizer_uri is not None):
83 + logger.warning(
84 + "Loading a model using Tensorizer with quantization on vLLM"
85 + " is unstable and may lead to errors.")
86 +
87 +
88 def load_with_tensorizer(tensorizer_config: TensorizerConfig,
89 **extra_kwargs) -> nn.Module:
90 tensorizer = TensorizerAgent(tensorizer_config, **extra_kwargs)
91 return tensorizer.deserialize()
92
93
46 -def tensorizer_warning(message: str):
47 - return warnings.warn(message, category=PerformanceWarning, stacklevel=2)
48 -
49 -
94 def is_vllm_serialized_tensorizer(tensorizer_config: TensorizerConfig) -> bool:
95 if tensorizer_config is None:
96 return False
97 return tensorizer_config.vllm_tensorized
98
99
56 -class ParameterizedLoadFormat(str):
57 - __slots__ = "params"
58 -
59 -
60 -class PerformanceWarning(UserWarning):
61 -
62 - def __str__(self):
63 - return (f"{super().__str__()}"
64 - " (set the VLLM_SILENCE_PERFORMANCE_WARNINGS"
65 - " environment variable to hide this)")
66 -
67 -
68 -if (os.getenv("VLLM_SILENCE_PERFORMANCE_WARNINGS", "").lower()
69 - not in ("", "0", "n", "no", "off", "disable")):
70 - warnings.simplefilter("ignore", category=PerformanceWarning)
71 -
72 -
100 @dataclass
101 class TensorizerArgs:
102 tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
@@ -219,11 +246,17 @@ class TensorizerAgent:
246 behavior of the TensorDeserializer when loading tensors from a serialized
247 model. For deserializations of HuggingFace models, TensorDeserializer is
248 instead used as an iterator directly in the func hf_model_weights_iterator
222 - in vllm/model_executor/weight_utils.py
249 + in vllm/model_executor/model_loader/weight_utils.py
250 """
251
252 def __init__(self, tensorizer_config: TensorizerConfig,
253 linear_method: LinearMethodBase, **extra_kwargs):
254 + if tensorizer_load_fail is not None:
255 + raise ImportError(
256 + "Tensorizer is not installed. Please install tensorizer "
257 + "to use this feature with `pip install vllm[tensorizer]`."
258 + ) from tensorizer_load_fail
259 +
260 self.tensorizer_config = tensorizer_config
261 self.tensorizer_args = (
262 self.tensorizer_config._construct_tensorizer_args())
@@ -234,11 +267,6 @@ class TensorizerAgent:
267 self.linear_method = linear_method
268 self.model = self._init_model()
269
237 - if tensorizer_load_fail:
238 - raise ImportError(
239 - "Tensorizer is not installed. Please install tensorizer "
240 - "to use this feature with `pip install vllm[tensorizer]`.")
241 -
270 def _init_model(self):
271 model_args = self.tensorizer_config.hf_config
272 model_args.torch_dtype = self.tensorizer_config.dtype
@@ -313,3 +341,23 @@ class TensorizerAgent:
341 self._check_tensors_on_meta_device()
342 self._resize_lora_embeddings()
343 return self.model.eval()
344 +
345 +
346 +def tensorizer_weights_iterator(
347 + tensorizer_args: "TensorizerArgs"
348 +) -> Generator[Tuple[str, torch.Tensor], None, None]:
349 + logger.warning(
350 + "Deserializing HuggingFace models is not optimized for "
351 + "loading on vLLM, as tensorizer is forced to load to CPU. "
352 + "Consider deserializing a vLLM model instead for faster "
353 + "load times. See the examples/tensorize_vllm_model.py example "
354 + "script for serializing vLLM models.")
355 +
356 + deserializer_args = tensorizer_args.deserializer_params
357 + stream_params = tensorizer_args.stream_params
358 + stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
359 + with TensorDeserializer(stream, **deserializer_args,
360 + device="cpu") as state:
361 + for name, param in state.items():
362 + yield name, param
363 + del state
vllm/model_executor/model_loader/utils.py new
+40
@@ -0,0 +1,40 @@
1 +"""Utilities for selecting and loading models."""
2 +import contextlib
3 +from typing import Tuple, Type
4 +
5 +import torch
6 +from torch import nn
7 +
8 +from vllm.config import ModelConfig
9 +from vllm.model_executor.models import ModelRegistry
10 +
11 +
12 +@contextlib.contextmanager
13 +def set_default_torch_dtype(dtype: torch.dtype):
14 + """Sets the default torch dtype to the given dtype."""
15 + old_dtype = torch.get_default_dtype()
16 + torch.set_default_dtype(dtype)
17 + yield
18 + torch.set_default_dtype(old_dtype)
19 +
20 +
21 +def get_model_architecture(
22 + model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
23 + architectures = getattr(model_config.hf_config, "architectures", [])
24 + # Special handling for quantized Mixtral.
25 + # FIXME(woosuk): This is a temporary hack.
26 + if (model_config.quantization is not None
27 + and "MixtralForCausalLM" in architectures):
28 + architectures = ["QuantMixtralForCausalLM"]
29 +
30 + for arch in architectures:
31 + model_cls = ModelRegistry.load_model_cls(arch)
32 + if model_cls is not None:
33 + return (model_cls, arch)
34 + raise ValueError(
35 + f"Model architectures {architectures} are not supported for now. "
36 + f"Supported architectures: {ModelRegistry.get_supported_archs()}")
37 +
38 +
39 +def get_architecture_class_name(model_config: ModelConfig) -> str:
40 + return get_model_architecture(model_config)[1]
vllm/model_executor/model_loader/weight_utils.py renamed
+127 -168
@@ -4,8 +4,9 @@ import glob
4 import hashlib
5 import json
6 import os
7 +import tempfile
8 from collections import defaultdict
8 -from typing import Any, Iterable, Iterator, List, Optional, Tuple, Union
9 +from typing import Any, Generator, Iterable, List, Optional, Tuple
10
11 import filelock
12 import huggingface_hub.constants
@@ -15,7 +16,7 @@ from huggingface_hub import HfFileSystem, snapshot_download
16 from safetensors.torch import load_file, safe_open, save_file
17 from tqdm.auto import tqdm
18
18 -from vllm.config import ModelConfig
19 +from vllm.config import LoadConfig, ModelConfig
20 from vllm.logger import init_logger
21 from vllm.model_executor.layers.quantization import (QuantizationConfig,
22 get_quantization_config)
@@ -27,8 +28,7 @@ logger = init_logger(__name__)
28 # can share the same lock without error.
29 # lock files in the temp directory will be automatically deleted when the
30 # system reboots, so users will not complain about annoying lock files
30 -temp_dir = os.environ.get('TMPDIR') or os.environ.get(
31 - 'TEMP') or os.environ.get('TMP') or "/tmp/"
31 +temp_dir = tempfile.gettempdir()
32
33
34 def enable_hf_transfer():
@@ -46,7 +46,7 @@ def enable_hf_transfer():
46 enable_hf_transfer()
47
48
49 -class Disabledtqdm(tqdm):
49 +class DisabledTqdm(tqdm):
50
51 def __init__(self, *args, **kwargs):
52 super().__init__(*args, **kwargs, disable=True)
@@ -114,7 +114,8 @@ def convert_bin_to_safetensor_file(
114
115
116 # TODO(woosuk): Move this to other place.
117 -def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
117 +def get_quant_config(model_config: ModelConfig,
118 + load_config: LoadConfig) -> QuantizationConfig:
119 quant_cls = get_quantization_config(model_config.quantization)
120 # Read the quantization config from the HF model config, if available.
121 hf_quant_config = getattr(model_config.hf_config, "quantization_config",
@@ -125,12 +126,12 @@ def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
126 is_local = os.path.isdir(model_name_or_path)
127 if not is_local:
128 # Download the config files.
128 - with get_lock(model_name_or_path, model_config.download_dir):
129 + with get_lock(model_name_or_path, load_config.download_dir):
130 hf_folder = snapshot_download(model_name_or_path,
131 revision=model_config.revision,
132 allow_patterns="*.json",
132 - cache_dir=model_config.download_dir,
133 - tqdm_class=Disabledtqdm)
133 + cache_dir=load_config.download_dir,
134 + tqdm_class=DisabledTqdm)
135 else:
136 hf_folder = model_name_or_path
137 config_files = glob.glob(os.path.join(hf_folder, "*.json"))
@@ -153,169 +154,127 @@ def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
154 return quant_cls.from_config(config)
155
156
156 -def prepare_hf_model_weights(
157 - model_name_or_path: str,
158 - cache_dir: Optional[str] = None,
159 - load_format: str = "auto",
160 - fall_back_to_pt: bool = True,
161 - revision: Optional[str] = None,
162 -) -> Tuple[str, List[str], bool]:
163 - # Download model weights from huggingface.
164 - is_local = os.path.isdir(model_name_or_path) \
165 - and load_format != "tensorizer"
166 - use_safetensors = False
167 - # Some quantized models use .pt files for storing the weights.
168 - if load_format == "auto":
169 - allow_patterns = ["*.safetensors", "*.bin"]
170 - elif load_format == "safetensors":
171 - use_safetensors = True
172 - allow_patterns = ["*.safetensors"]
173 - elif load_format == "pt":
174 - allow_patterns = ["*.pt"]
175 - elif load_format == "npcache":
176 - allow_patterns = ["*.bin"]
177 - elif load_format == "tensorizer":
178 - allow_patterns = ["*.tensors"]
179 - else:
180 - raise ValueError(f"Unknown load_format: {load_format}")
181 -
182 - if fall_back_to_pt:
183 - allow_patterns += ["*.pt"]
184 -
185 - if not is_local and load_format != "tensorizer":
186 - # Before we download we look at that is available:
187 - fs = HfFileSystem()
188 - file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
189 -
190 - # depending on what is available we download different things
191 - for pattern in allow_patterns:
192 - matching = fnmatch.filter(file_list, pattern)
193 - if len(matching) > 0:
194 - allow_patterns = [pattern]
195 - break
196 -
197 - logger.info(f"Using model weights format {allow_patterns}")
198 - # Use file lock to prevent multiple processes from
199 - # downloading the same model weights at the same time.
200 - with get_lock(model_name_or_path, cache_dir):
201 - hf_folder = snapshot_download(model_name_or_path,
202 - allow_patterns=allow_patterns,
203 - cache_dir=cache_dir,
204 - tqdm_class=Disabledtqdm,
205 - revision=revision)
206 - else:
207 - hf_folder = model_name_or_path
208 - hf_weights_files: List[str] = []
157 +def download_weights_from_hf(model_name_or_path: str,
158 + cache_dir: Optional[str],
159 + allow_patterns: List[str],
160 + revision: Optional[str] = None) -> str:
161 + """Download model weights from Hugging Face Hub.
162 +
163 + Args:
164 + model_name_or_path (str): The model name or path.
165 + cache_dir (Optional[str]): The cache directory to store the model
166 + weights. If None, will use HF defaults.
167 + allow_patterns (List[str]): The allowed patterns for the
168 + weight files. Files matched by any of the patterns will be
169 + downloaded.
170 + revision (Optional[str]): The revision of the model.
171 +
172 + Returns:
173 + str: The path to the downloaded model weights.
174 + """
175 + # Before we download we look at that is available:
176 + fs = HfFileSystem()
177 + file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
178 +
179 + # depending on what is available we download different things
180 for pattern in allow_patterns:
210 - hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
211 - if len(hf_weights_files) > 0:
212 - if pattern == "*.safetensors":
213 - use_safetensors = True
181 + matching = fnmatch.filter(file_list, pattern)
182 + if len(matching) > 0:
183 + allow_patterns = [pattern]
184 break
215 - if not use_safetensors:
216 - # Exclude files that are not needed for inference.
217 - # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
218 - blacklist = [
219 - "training_args.bin",
220 - "optimizer.bin",
221 - "optimizer.pt",
222 - "scheduler.pt",
223 - "scaler.pt",
224 - ]
225 - hf_weights_files = [
226 - f for f in hf_weights_files
227 - if not any(f.endswith(x) for x in blacklist)
228 - ]
229 -
230 - if load_format == "tensorizer":
231 - return hf_folder, hf_weights_files, use_safetensors
232 -
233 - if len(hf_weights_files) == 0:
234 - raise RuntimeError(
235 - f"Cannot find any model weights with `{model_name_or_path}`")
236 -
237 - return hf_folder, hf_weights_files, use_safetensors
238 -
239 -
240 -def hf_model_weights_iterator(
241 - model_name_or_path: str,
242 - cache_dir: Optional[str] = None,
243 - load_format: Union[Tuple, str] = "auto",
244 - revision: Optional[str] = None,
245 - fall_back_to_pt: Optional[bool] = True,
246 -) -> Iterator[Tuple[str, torch.Tensor]]:
247 - hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(
248 - model_name_or_path,
249 - cache_dir=cache_dir,
250 - load_format=load_format,
251 - fall_back_to_pt=fall_back_to_pt,
252 - revision=revision)
253 -
254 - if load_format == "npcache":
255 - # Currently np_cache only support *.bin checkpoints
256 - assert use_safetensors is False
257 -
258 - # Convert the model weights from torch tensors to numpy arrays for
259 - # faster loading.
260 - np_folder = os.path.join(hf_folder, "np")
261 - os.makedirs(np_folder, exist_ok=True)
262 - weight_names_file = os.path.join(np_folder, "weight_names.json")
263 - # Use file lock to prevent multiple processes from
264 - # dumping the same model weights to numpy at the same time.
265 - with get_lock(model_name_or_path, cache_dir):
266 - if not os.path.exists(weight_names_file):
267 - weight_names = []
268 - for bin_file in hf_weights_files:
269 - state = torch.load(bin_file, map_location="cpu")
270 - for name, param in state.items():
271 - param_path = os.path.join(np_folder, name)
272 - with open(param_path, "wb") as f:
273 - np.save(f, param.cpu().detach().numpy())
274 - weight_names.append(name)
275 - with open(weight_names_file, "w") as f:
276 - json.dump(weight_names, f)
277 -
278 - with open(weight_names_file, "r") as f:
279 - weight_names = json.load(f)
280 -
281 - for name in weight_names:
282 - param_path = os.path.join(np_folder, name)
283 - with open(param_path, "rb") as f:
284 - param = np.load(f)
285 - yield name, torch.from_numpy(param)
286 - elif load_format == "tensorizer":
287 - from vllm.model_executor.tensorizer_loader import (TensorDeserializer,
288 - open_stream,
289 - tensorizer_warning)
290 - tensorizer_args = load_format.params
291 - tensorizer_warning(
292 - "Deserializing HuggingFace models is not optimized for "
293 - "loading on vLLM, as tensorizer is forced to load to CPU. "
294 - "Consider deserializing a vLLM model instead for faster "
295 - "load times. See the examples/tensorize_vllm_model.py example "
296 - "script for serializing vLLM models.")
297 -
298 - deserializer_args = tensorizer_args.deserializer_params
299 - stream_params = tensorizer_args.stream_params
300 - stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
301 - with TensorDeserializer(stream, **deserializer_args,
302 - device="cpu") as state:
303 - for name, param in state.items():
185 +
186 + logger.info(f"Using model weights format {allow_patterns}")
187 + # Use file lock to prevent multiple processes from
188 + # downloading the same model weights at the same time.
189 + with get_lock(model_name_or_path, cache_dir):
190 + hf_folder = snapshot_download(model_name_or_path,
191 + allow_patterns=allow_patterns,
192 + cache_dir=cache_dir,
193 + tqdm_class=DisabledTqdm,
194 + revision=revision)
195 + return hf_folder
196 +
197 +
198 +def filter_files_not_needed_for_inference(
199 + hf_weights_files: List[str]) -> List[str]:
200 + """
201 + Exclude files that are not needed for inference.
202 +
203 + See https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
204 + """
205 + blacklist = [
206 + "training_args.bin",
207 + "optimizer.bin",
208 + "optimizer.pt",
209 + "scheduler.pt",
210 + "scaler.pt",
211 + ]
212 + hf_weights_files = [
213 + f for f in hf_weights_files
214 + if not any(f.endswith(x) for x in blacklist)
215 + ]
216 + return hf_weights_files
217 +
218 +
219 +def np_cache_weights_iterator(
220 + model_name_or_path: str, cache_dir: Optional[str], hf_folder: str,
221 + hf_weights_files: List[str]
222 +) -> Generator[Tuple[str, torch.Tensor], None, None]:
223 + """Iterate over the weights in the model np files.
224 +
225 + Will dump the model weights to numpy files if they are not already dumped.
226 + """
227 + # Convert the model weights from torch tensors to numpy arrays for
228 + # faster loading.
229 + np_folder = os.path.join(hf_folder, "np")
230 + os.makedirs(np_folder, exist_ok=True)
231 + weight_names_file = os.path.join(np_folder, "weight_names.json")
232 + # Use file lock to prevent multiple processes from
233 + # dumping the same model weights to numpy at the same time.
234 + with get_lock(model_name_or_path, cache_dir):
235 + if not os.path.exists(weight_names_file):
236 + weight_names = []
237 + for bin_file in hf_weights_files:
238 + state = torch.load(bin_file, map_location="cpu")
239 + for name, param in state.items():
240 + param_path = os.path.join(np_folder, name)
241 + with open(param_path, "wb") as f:
242 + np.save(f, param.cpu().detach().numpy())
243 + weight_names.append(name)
244 + with open(weight_names_file, "w") as f:
245 + json.dump(weight_names, f)
246 +
247 + with open(weight_names_file, "r") as f:
248 + weight_names = json.load(f)
249 +
250 + for name in weight_names:
251 + param_path = os.path.join(np_folder, name)
252 + with open(param_path, "rb") as f:
253 + param = np.load(f)
254 + yield name, torch.from_numpy(param)
255 +
256 +
257 +def safetensors_weights_iterator(
258 + hf_weights_files: List[str]
259 +) -> Generator[Tuple[str, torch.Tensor], None, None]:
260 + """Iterate over the weights in the model safetensor files."""
261 + for st_file in hf_weights_files:
262 + with safe_open(st_file, framework="pt") as f:
263 + for name in f.keys(): # noqa: SIM118
264 + param = f.get_tensor(name)
265 yield name, param
266 +
267 +
268 +def pt_weights_iterator(
269 + hf_weights_files: List[str]
270 +) -> Generator[Tuple[str, torch.Tensor], None, None]:
271 + """Iterate over the weights in the model bin/pt files."""
272 + for bin_file in hf_weights_files:
273 + state = torch.load(bin_file, map_location="cpu")
274 + for name, param in state.items():
275 + yield name, param
276 del state
306 - elif use_safetensors:
307 - for st_file in hf_weights_files:
308 - with safe_open(st_file, framework="pt") as f:
309 - for name in f.keys(): # noqa: SIM118
310 - param = f.get_tensor(name)
311 - yield name, param
312 - else:
313 - for bin_file in hf_weights_files:
314 - state = torch.load(bin_file, map_location="cpu")
315 - for name, param in state.items():
316 - yield name, param
317 - del state
318 - torch.cuda.empty_cache()
277 + torch.cuda.empty_cache()
278
279
280 def kv_cache_scales_loader(
vllm/model_executor/models/baichuan.py
+4 -10
@@ -19,7 +19,7 @@
19 # limitations under the License.
20 """Inference-only BaiChuan model compatible with HuggingFace weights."""
21 import math
22 -from typing import List, Optional, Tuple
22 +from typing import Iterable, List, Optional, Tuple
23
24 import torch
25 from torch import nn
@@ -40,9 +40,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
43 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 -from vllm.model_executor.weight_utils import (default_weight_loader,
45 - hf_model_weights_iterator)
45 from vllm.sequence import SamplerOutput
46
47
@@ -340,19 +339,14 @@ class BaiChuanBaseForCausalLM(nn.Module):
339 next_tokens = self.sampler(logits, sampling_metadata)
340 return next_tokens
341
343 - def load_weights(self,
344 - model_name_or_path: str,
345 - cache_dir: Optional[str] = None,
346 - load_format: str = "auto",
347 - revision: Optional[str] = None):
342 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
343 stacked_params_mapping = [
344 # (param_name, shard_name, shard_id)
345 ("gate_up_proj", "gate_proj", 0),
346 ("gate_up_proj", "up_proj", 1),
347 ]
348 params_dict = dict(self.named_parameters())
354 - for name, loaded_weight in hf_model_weights_iterator(
355 - model_name_or_path, cache_dir, load_format, revision):
349 + for name, loaded_weight in weights:
350 if "rotary_emb.inv_freq" in name:
351 continue
352 if name == "lm_head.weight":
vllm/model_executor/models/bloom.py
+4 -10
@@ -17,7 +17,7 @@
17 # limitations under the License.
18 """Inference-only BLOOM model compatible with HuggingFace weights."""
19 import math
20 -from typing import List, Optional
20 +from typing import Iterable, List, Optional, Tuple
21
22 import torch
23 from torch import nn
@@ -35,9 +35,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
38 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 -from vllm.model_executor.weight_utils import (default_weight_loader,
40 - hf_model_weights_iterator)
40 from vllm.sequence import SamplerOutput
41
42
@@ -298,14 +297,9 @@ class BloomForCausalLM(nn.Module):
297 next_tokens = self.sampler(logits, sampling_metadata)
298 return next_tokens
299
301 - def load_weights(self,
302 - model_name_or_path: str,
303 - cache_dir: Optional[str] = None,
304 - load_format: str = "auto",
305 - revision: Optional[str] = None):
300 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
301 params_dict = dict(self.named_parameters(remove_duplicate=False))
307 - for name, loaded_weight in hf_model_weights_iterator(
308 - model_name_or_path, cache_dir, load_format, revision):
302 + for name, loaded_weight in weights:
303 if name == "lm_head.weight":
304 continue
305 if not name.startswith("transformer."):
vllm/model_executor/models/chatglm.py
+4 -10
@@ -2,7 +2,7 @@
2 # Adapted from
3 # https://github.com/THUDM/ChatGLM2-6B
4 """Inference-only ChatGLM model compatible with THUDM weights."""
5 -from typing import List, Optional
5 +from typing import Iterable, List, Optional, Tuple
6
7 import torch
8 from torch import nn
@@ -22,9 +22,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22 from vllm.model_executor.layers.sampler import Sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 ParallelLMHead, VocabParallelEmbedding)
25 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 -from vllm.model_executor.weight_utils import (default_weight_loader,
27 - hf_model_weights_iterator)
27 from vllm.sequence import SamplerOutput
28 from vllm.transformers_utils.configs import ChatGLMConfig
29
@@ -370,14 +369,9 @@ class ChatGLMForCausalLM(nn.Module):
369 next_tokens = self.sampler(logits, sampling_metadata)
370 return next_tokens
371
373 - def load_weights(self,
374 - model_name_or_path: str,
375 - cache_dir: Optional[str] = None,
376 - load_format: str = "auto",
377 - revision: Optional[str] = None):
372 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
373 params_dict = dict(self.named_parameters(remove_duplicate=False))
379 - for name, loaded_weight in hf_model_weights_iterator(
380 - model_name_or_path, cache_dir, load_format, revision):
374 + for name, loaded_weight in weights:
375 if "rotary_pos_emb.inv_freq" in name:
376 continue
377 if "word_embeddings" in name:
vllm/model_executor/models/commandr.py
+4 -12
@@ -20,7 +20,7 @@
20
21 # This file is based on the LLama model definition file in transformers
22 """PyTorch Cohere model."""
23 -from typing import List, Optional, Tuple
23 +from typing import Iterable, List, Optional, Tuple
24
25 import torch
26 import torch.utils.checkpoint
@@ -41,10 +41,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 VocabParallelEmbedding)
44 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.model_executor.utils import set_weight_attrs
46 -from vllm.model_executor.weight_utils import (default_weight_loader,
47 - hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
48
49
@@ -335,13 +334,7 @@ class CohereForCausalLM(nn.Module):
334 next_tokens = self.sampler(logits, sampling_metadata)
335 return next_tokens
336
338 - def load_weights(
339 - self,
340 - model_name_or_path: str,
341 - cache_dir: Optional[str] = None,
342 - load_format: str = "auto",
343 - revision: Optional[str] = None,
344 - ):
337 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
338 stacked_params_mapping = [
339 # (param_name, shard_name, shard_id)
340 ("qkv_proj", "q_proj", "q"),
@@ -352,8 +345,7 @@ class CohereForCausalLM(nn.Module):
345 ]
346 params_dict = dict(self.named_parameters())
347 loaded_params = set()
355 - for name, loaded_weight in hf_model_weights_iterator(
356 - model_name_or_path, cache_dir, load_format, revision):
348 + for name, loaded_weight in weights:
349 for param_name, shard_name, shard_id in stacked_params_mapping:
350 if shard_name not in name:
351 continue
vllm/model_executor/models/dbrx.py
+4 -12
@@ -1,5 +1,5 @@
1 # coding=utf-8
2 -from typing import List, Optional
2 +from typing import Iterable, List, Optional, Tuple
3
4 import torch
5 import torch.nn as nn
@@ -18,10 +18,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
21 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.model_executor.utils import set_weight_attrs
23 -from vllm.model_executor.weight_utils import (default_weight_loader,
24 - hf_model_weights_iterator)
24 from vllm.sequence import SamplerOutput
25 from vllm.transformers_utils.configs.dbrx import DbrxConfig
26
@@ -391,20 +390,13 @@ class DbrxForCausalLM(nn.Module):
390 next_tokens = self.sampler(logits, sampling_metadata)
391 return next_tokens
392
394 - def load_weights(
395 - self,
396 - model_name_or_path: str,
397 - cache_dir: Optional[str] = None,
398 - load_format: str = "auto",
399 - revision: Optional[str] = None,
400 - ):
393 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
394 expert_params_mapping = [(
395 "ws" if weight_name in ["w1", "v1"] else "w2s",
396 f"experts.mlp.{weight_name}",
397 ) for weight_name in ["w1", "v1", "w2"]]
398 params_dict = dict(self.named_parameters(remove_duplicate=False))
406 - for name, loaded_weight in hf_model_weights_iterator(
407 - model_name_or_path, cache_dir, load_format, revision):
399 + for name, loaded_weight in weights:
400 for param_name, weight_name in expert_params_mapping:
401 if weight_name not in name:
402 continue
vllm/model_executor/models/decilm.py
+4 -10
@@ -23,16 +23,15 @@
23 # limitations under the License.
24 """Inference-only DeciLM model compatible with HuggingFace weights."""
25
26 -from typing import Optional
26 +from typing import Iterable, Optional, Tuple
27
28 import torch
29 from transformers import PretrainedConfig
30
31 from vllm.config import LoRAConfig
32 from vllm.model_executor.layers.linear import LinearMethodBase
33 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34 from vllm.model_executor.models.llama import LlamaForCausalLM
34 -from vllm.model_executor.weight_utils import (default_weight_loader,
35 - hf_model_weights_iterator)
35
36
37 class DeciLMForCausalLM(LlamaForCausalLM):
@@ -65,11 +64,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
64 linear_method=linear_method,
65 lora_config=lora_config)
66
68 - def load_weights(self,
69 - model_name_or_path: str,
70 - cache_dir: Optional[str] = None,
71 - load_format: str = "auto",
72 - revision: Optional[str] = None):
67 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
68 stacked_params_mapping = [
69 # (param_name, shard_name, shard_id)
70 ("qkv_proj", "q_proj", "q"),
@@ -79,8 +74,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
74 ("gate_up_proj", "up_proj", 1),
75 ]
76 params_dict = dict(self.named_parameters())
82 - for name, loaded_weight in hf_model_weights_iterator(
83 - model_name_or_path, cache_dir, load_format, revision):
77 + for name, loaded_weight in weights:
78 if "rotary_emb.inv_freq" in name:
79 continue
80
vllm/model_executor/models/deepseek.py
+6 -14
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Deepseek model."""
24 -from typing import Any, Dict, List, Optional
24 +from typing import Any, Dict, Iterable, List, Optional, Tuple
25
26 import torch
27 from torch import nn
@@ -44,9 +44,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
44 from vllm.model_executor.layers.sampler import Sampler
45 from vllm.model_executor.layers.vocab_parallel_embedding import (
46 ParallelLMHead, VocabParallelEmbedding)
47 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 -from vllm.model_executor.weight_utils import (default_weight_loader,
49 - hf_model_weights_iterator)
49 from vllm.sequence import SamplerOutput
50
51
@@ -316,6 +315,8 @@ class DeepseekDecoderLayer(nn.Module):
315
316 class DeepseekModel(nn.Module):
317
318 + fall_back_to_pt_during_load = False
319 +
320 def __init__(
321 self,
322 config: PretrainedConfig,
@@ -395,11 +396,7 @@ class DeepseekForCausalLM(nn.Module):
396 next_tokens = self.sampler(logits, sampling_metadata)
397 return next_tokens
398
398 - def load_weights(self,
399 - model_name_or_path: str,
400 - cache_dir: Optional[str] = None,
401 - load_format: str = "auto",
402 - revision: Optional[str] = None):
399 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
400 stacked_params_mapping = [
401 # (param_name, shard_name, shard_id)
402 ("qkv_proj", "q_proj", "q"),
@@ -410,12 +407,7 @@ class DeepseekForCausalLM(nn.Module):
407 ]
408
409 params_dict = dict(self.named_parameters())
413 - for name, loaded_weight in hf_model_weights_iterator(
414 - model_name_or_path,
415 - cache_dir,
416 - load_format,
417 - revision,
418 - fall_back_to_pt=False):
410 + for name, loaded_weight in weights:
411 if "rotary_emb.inv_freq" in name:
412 continue
413 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/falcon.py
+4 -10
@@ -19,7 +19,7 @@
19 """PyTorch Falcon model."""
20
21 import math
22 -from typing import List, Optional, Union
22 +from typing import Iterable, List, Optional, Tuple, Union
23
24 import torch
25 from torch import nn
@@ -40,9 +40,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 VocabParallelEmbedding)
43 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 -from vllm.model_executor.weight_utils import (default_weight_loader,
45 - hf_model_weights_iterator)
45 from vllm.sequence import SamplerOutput
46 from vllm.transformers_utils.configs import RWConfig
47
@@ -399,11 +398,7 @@ class FalconForCausalLM(nn.Module):
398 next_tokens = self.sampler(logits, sampling_metadata)
399 return next_tokens
400
402 - def load_weights(self,
403 - model_name_or_path: str,
404 - cache_dir: Optional[str] = None,
405 - load_format: str = "auto",
406 - revision: Optional[str] = None):
401 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
402 total_num_heads = self.config.num_attention_heads
403 if self.config.new_decoder_architecture:
404 total_num_kv_heads = self.config.num_kv_heads
@@ -413,8 +408,7 @@ class FalconForCausalLM(nn.Module):
408 total_num_kv_heads = total_num_heads
409 num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
410 params_dict = dict(self.named_parameters(remove_duplicate=False))
416 - for name, loaded_weight in hf_model_weights_iterator(
417 - model_name_or_path, cache_dir, load_format, revision):
411 + for name, loaded_weight in weights:
412 if name == "lm_head.weight":
413 # Falcon uses tied embeddings.
414 continue
vllm/model_executor/models/gemma.py
+4 -10
@@ -15,7 +15,7 @@
15 # limitations under the License.
16 """Inference-only Gemma model compatible with HuggingFace weights."""
17 from functools import lru_cache
18 -from typing import List, Optional, Tuple
18 +from typing import Iterable, List, Optional, Tuple
19
20 import torch
21 from torch import nn
@@ -36,9 +36,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
39 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 -from vllm.model_executor.weight_utils import (default_weight_loader,
41 - hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42
43 logger = init_logger(__name__)
@@ -346,11 +345,7 @@ class GemmaForCausalLM(nn.Module):
345 next_tokens = self.sampler(logits, sampling_metadata)
346 return next_tokens
347
349 - def load_weights(self,
350 - model_name_or_path: str,
351 - cache_dir: Optional[str] = None,
352 - load_format: str = "auto",
353 - revision: Optional[str] = None):
348 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
349 stacked_params_mapping = [
350 # (param_name, shard_name, shard_id)
351 ("qkv_proj", "q_proj", "q"),
@@ -361,8 +356,7 @@ class GemmaForCausalLM(nn.Module):
356 ]
357 params_dict = dict(self.named_parameters())
358 loaded_params = set()
364 - for name, loaded_weight in hf_model_weights_iterator(
365 - model_name_or_path, cache_dir, load_format, revision):
359 + for name, loaded_weight in weights:
360 for (param_name, shard_name, shard_id) in stacked_params_mapping:
361 if shard_name not in name:
362 continue
vllm/model_executor/models/gpt2.py
+4 -10
@@ -17,7 +17,7 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 """Inference-only GPT-2 model compatible with HuggingFace weights."""
20 -from typing import List, Optional
20 +from typing import Iterable, List, Optional, Tuple
21
22 import torch
23 from torch import nn
@@ -34,9 +34,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding)
37 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 -from vllm.model_executor.weight_utils import (default_weight_loader,
39 - hf_model_weights_iterator)
39 from vllm.sequence import SamplerOutput
40
41
@@ -239,14 +238,9 @@ class GPT2LMHeadModel(nn.Module):
238 next_tokens = self.sampler(logits, sampling_metadata)
239 return next_tokens
240
242 - def load_weights(self,
243 - model_name_or_path: str,
244 - cache_dir: Optional[str] = None,
245 - load_format: str = "auto",
246 - revision: Optional[str] = None):
241 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
242 params_dict = dict(self.named_parameters(remove_duplicate=False))
248 - for name, loaded_weight in hf_model_weights_iterator(
249 - model_name_or_path, cache_dir, load_format, revision):
243 + for name, loaded_weight in weights:
244 if "lm_head.weight" in name:
245 # GPT-2 ties the weights of the embedding layer and the final
246 # linear layer.
vllm/model_executor/models/gpt_bigcode.py
+4 -10
@@ -18,7 +18,7 @@
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 """Inference-only GPTBigCode model compatible with HuggingFace weights."""
21 -from typing import List, Optional
21 +from typing import Iterable, List, Optional, Tuple
22
23 import torch
24 from torch import nn
@@ -35,9 +35,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
38 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 -from vllm.model_executor.weight_utils import (default_weight_loader,
40 - hf_model_weights_iterator)
40 from vllm.sequence import SamplerOutput
41
42
@@ -260,14 +259,9 @@ class GPTBigCodeForCausalLM(nn.Module):
259 next_tokens = self.sampler(logits, sampling_metadata)
260 return next_tokens
261
263 - def load_weights(self,
264 - model_name_or_path: str,
265 - cache_dir: Optional[str] = None,
266 - load_format: str = "auto",
267 - revision: Optional[str] = None):
262 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
263 params_dict = dict(self.named_parameters(remove_duplicate=False))
269 - for name, loaded_weight in hf_model_weights_iterator(
270 - model_name_or_path, cache_dir, load_format, revision):
264 + for name, loaded_weight in weights:
265 if "lm_head.weight" in name:
266 continue
267 if ".attn.bias" in name:
vllm/model_executor/models/gpt_j.py
+4 -10
@@ -16,7 +16,7 @@
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 """Inference-only GPT-J model compatible with HuggingFace weights."""
19 -from typing import List, Optional
19 +from typing import Iterable, List, Optional, Tuple
20
21 import torch
22 from torch import nn
@@ -34,9 +34,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 ParallelLMHead, VocabParallelEmbedding)
37 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 -from vllm.model_executor.weight_utils import (default_weight_loader,
39 - hf_model_weights_iterator)
39 from vllm.sequence import SamplerOutput
40
41
@@ -248,11 +247,7 @@ class GPTJForCausalLM(nn.Module):
247 next_tokens = self.sampler(logits, sampling_metadata)
248 return next_tokens
249
251 - def load_weights(self,
252 - model_name_or_path: str,
253 - cache_dir: Optional[str] = None,
254 - load_format: str = "auto",
255 - revision: Optional[str] = None):
250 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
251 stacked_params_mapping = [
252 # (param_name, shard_name, shard_id)
253 ("qkv_proj", "q_proj", "q"),
@@ -262,8 +257,7 @@ class GPTJForCausalLM(nn.Module):
257 ("gate_up_proj", "up_proj", 1),
258 ]
259 params_dict = dict(self.named_parameters())
265 - for name, loaded_weight in hf_model_weights_iterator(
266 - model_name_or_path, cache_dir, load_format, revision):
260 + for name, loaded_weight in weights:
261 if "attn.bias" in name or "attn.masked_bias" in name:
262 continue
263 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/gpt_neox.py
+4 -10
@@ -16,7 +16,7 @@
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 """Inference-only GPT-NeoX model compatible with HuggingFace weights."""
19 -from typing import List, Optional
19 +from typing import Iterable, List, Optional, Tuple
20
21 import torch
22 from torch import nn
@@ -34,9 +34,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 ParallelLMHead, VocabParallelEmbedding)
37 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 -from vllm.model_executor.weight_utils import (default_weight_loader,
39 - hf_model_weights_iterator)
39 from vllm.sequence import SamplerOutput
40
41
@@ -262,14 +261,9 @@ class GPTNeoXForCausalLM(nn.Module):
261 next_tokens = self.sampler(logits, sampling_metadata)
262 return next_tokens
263
265 - def load_weights(self,
266 - model_name_or_path: str,
267 - cache_dir: Optional[str] = None,
268 - load_format: str = "auto",
269 - revision: Optional[str] = None):
264 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
265 params_dict = dict(self.named_parameters())
271 - for name, loaded_weight in hf_model_weights_iterator(
272 - model_name_or_path, cache_dir, load_format, revision):
266 + for name, loaded_weight in weights:
267 if ("attention.bias" in name or "attention.masked_bias" in name
268 or "rotary_emb.inv_freq" in name):
269 continue
vllm/model_executor/models/internlm2.py
+4 -10
@@ -1,5 +1,5 @@
1 # -*- coding: utf-8 -*-
2 -from typing import Any, Dict, List, Optional, Tuple
2 +from typing import Any, Dict, Iterable, List, Optional, Tuple
3
4 import torch
5 from torch import nn
@@ -18,9 +18,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 ParallelLMHead, VocabParallelEmbedding)
21 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 -from vllm.model_executor.weight_utils import (default_weight_loader,
23 - hf_model_weights_iterator)
23 from vllm.sequence import SamplerOutput
24
25
@@ -274,19 +273,14 @@ class InternLM2ForCausalLM(nn.Module):
273 next_tokens = self.sampler(logits, sampling_metadata)
274 return next_tokens
275
277 - def load_weights(self,
278 - model_name_or_path: str,
279 - cache_dir: Optional[str] = None,
280 - load_format: str = "auto",
281 - revision: Optional[str] = None):
276 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
277 stacked_params_mapping = [
278 # (param_name, shard_name, shard_id)
279 ("gate_up_proj", "w1", 0),
280 ("gate_up_proj", "w3", 1),
281 ]
282 params_dict = dict(self.named_parameters())
288 - for name, loaded_weight in hf_model_weights_iterator(
289 - model_name_or_path, cache_dir, load_format, revision):
283 + for name, loaded_weight in weights:
284 if "rotary_emb.inv_freq" in name:
285 continue
286 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/jais.py
+4 -12
@@ -20,7 +20,7 @@
20 """Inference-only Jais model compatible with HuggingFace weights."""
21
22 import math
23 -from typing import List, Optional
23 +from typing import Iterable, List, Optional, Tuple
24
25 import torch
26 from torch import nn
@@ -36,9 +36,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
39 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 -from vllm.model_executor.weight_utils import (default_weight_loader,
41 - hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42 from vllm.transformers_utils.configs import JAISConfig
43
@@ -303,16 +302,9 @@ class JAISLMHeadModel(nn.Module):
302 next_tokens = self.sampler(logits, sampling_metadata)
303 return next_tokens
304
306 - def load_weights(
307 - self,
308 - model_name_or_path: str,
309 - cache_dir: Optional[str] = None,
310 - load_format: str = "auto",
311 - revision: Optional[str] = None,
312 - ):
305 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
306 params_dict = dict(self.named_parameters(remove_duplicate=False))
314 - for name, loaded_weight in hf_model_weights_iterator(
315 - model_name_or_path, cache_dir, load_format, revision):
307 + for name, loaded_weight in weights:
308 if "lm_head.weight" in name:
309 # GPT-2 ties the weights of the embedding layer and the final
310 # linear layer.
vllm/model_executor/models/llama.py
+5 -11
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only LLaMA model compatible with HuggingFace weights."""
24 -from typing import Any, Dict, List, Optional, Tuple
24 +from typing import Any, Dict, Iterable, List, Optional, Tuple
25
26 import torch
27 from torch import nn
@@ -42,10 +42,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 +from vllm.model_executor.model_loader.weight_utils import (
46 + default_weight_loader, kv_cache_scales_loader)
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.model_executor.weight_utils import (default_weight_loader,
47 - hf_model_weights_iterator,
48 - kv_cache_scales_loader)
48 from vllm.sequence import SamplerOutput
49 from vllm.utils import is_hip
50
@@ -376,11 +375,7 @@ class LlamaForCausalLM(nn.Module):
375 next_tokens = self.sampler(logits, sampling_metadata)
376 return next_tokens
377
379 - def load_weights(self,
380 - model_name_or_path: str,
381 - cache_dir: Optional[str] = None,
382 - load_format: str = "auto",
383 - revision: Optional[str] = None):
378 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
379 stacked_params_mapping = [
380 # (param_name, shard_name, shard_id)
381 ("qkv_proj", "q_proj", "q"),
@@ -390,8 +385,7 @@ class LlamaForCausalLM(nn.Module):
385 ("gate_up_proj", "up_proj", 1),
386 ]
387 params_dict = dict(self.named_parameters())
393 - for name, loaded_weight in hf_model_weights_iterator(
394 - model_name_or_path, cache_dir, load_format, revision):
388 + for name, loaded_weight in weights:
389 if "rotary_emb.inv_freq" in name:
390 continue
391 if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/llava.py
+4 -10
@@ -1,4 +1,4 @@
1 -from typing import List, Optional
1 +from typing import Iterable, List, Optional, Tuple
2
3 import torch
4 from torch import nn
@@ -13,10 +13,9 @@ from vllm.model_executor.layers.linear import LinearMethodBase
13 from vllm.model_executor.layers.logits_processor import LogitsProcessor
14 from vllm.model_executor.layers.sampler import Sampler
15 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
16 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.model_executor.models.llama import LlamaModel
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 -from vllm.model_executor.weight_utils import (default_weight_loader,
19 - hf_model_weights_iterator)
19 from vllm.sequence import SamplerOutput
20
21 _KEYS_TO_MODIFY_MAPPING = {
@@ -198,11 +197,7 @@ class LlavaForConditionalGeneration(nn.Module):
197 next_tokens = self.sampler(logits, sampling_metadata)
198 return next_tokens
199
201 - def load_weights(self,
202 - model_name_or_path: str,
203 - cache_dir: Optional[str] = None,
204 - load_format: str = "auto",
205 - revision: Optional[str] = None):
200 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
201 # only doing this for language model part for now.
202 stacked_params_mapping = [
203 # (param_name, shard_name, shard_id)
@@ -213,8 +208,7 @@ class LlavaForConditionalGeneration(nn.Module):
208 ("gate_up_proj", "up_proj", 1),
209 ]
210 params_dict = dict(self.named_parameters())
216 - for name, loaded_weight in hf_model_weights_iterator(
217 - model_name_or_path, cache_dir, load_format, revision):
211 + for name, loaded_weight in weights:
212 if "rotary_emb.inv_freq" in name:
213 continue
214 for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
vllm/model_executor/models/minicpm.py
+4 -10
@@ -22,7 +22,7 @@
22 # limitations under the License.
23 """Inference-only MiniCPM model compatible with HuggingFace weights."""
24 import math
25 -from typing import Any, Dict, List, Optional, Tuple
25 +from typing import Any, Dict, Iterable, List, Optional, Tuple
26
27 import torch
28 from torch import nn
@@ -45,10 +45,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
45 from vllm.model_executor.layers.sampler import Sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.model_executor.utils import set_weight_attrs
50 -from vllm.model_executor.weight_utils import (default_weight_loader,
51 - hf_model_weights_iterator)
51 from vllm.sequence import SamplerOutput
52
53
@@ -472,11 +471,7 @@ class MiniCPMForCausalLM(nn.Module):
471 next_tokens = self.sampler(logits, sampling_metadata)
472 return next_tokens
473
475 - def load_weights(self,
476 - model_name_or_path: str,
477 - cache_dir: Optional[str] = None,
478 - load_format: str = "auto",
479 - revision: Optional[str] = None):
474 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
475 stacked_params_mapping = [
476 # (param_name, shard_name, shard_id)
477 ("qkv_proj", "q_proj", "q"),
@@ -493,8 +488,7 @@ class MiniCPMForCausalLM(nn.Module):
488 for weight_name in ["w1", "w2", "w3"]
489 ]
490 params_dict = dict(self.named_parameters())
496 - for name, loaded_weight in hf_model_weights_iterator(
497 - model_name_or_path, cache_dir, load_format, revision):
491 + for name, loaded_weight in weights:
492 if "rotary_emb.inv_freq" in name:
493 continue
494 if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/mixtral.py
+6 -14
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Mixtral model."""
24 -from typing import List, Optional
24 +from typing import Iterable, List, Optional, Tuple
25
26 import torch
27 from torch import nn
@@ -43,10 +43,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
46 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.model_executor.utils import set_weight_attrs
48 -from vllm.model_executor.weight_utils import (default_weight_loader,
49 - hf_model_weights_iterator)
49 from vllm.sequence import SamplerOutput
50
51
@@ -319,6 +318,8 @@ class MixtralModel(nn.Module):
318
319
320 class MixtralForCausalLM(nn.Module):
321 + fall_back_to_pt_during_load = False
322 +
323 packed_modules_mapping = {
324 "qkv_proj": [
325 "q_proj",
@@ -393,11 +394,7 @@ class MixtralForCausalLM(nn.Module):
394 next_tokens = self.sampler(logits, sampling_metadata)
395 return next_tokens
396
396 - def load_weights(self,
397 - model_name_or_path: str,
398 - cache_dir: Optional[str] = None,
399 - load_format: str = "auto",
400 - revision: Optional[str] = None):
397 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
398 stacked_params_mapping = [
399 # (param_name, shard_name, shard_id)
400 ("qkv_proj", "q_proj", "q"),
@@ -414,12 +411,7 @@ class MixtralForCausalLM(nn.Module):
411 ]
412
413 params_dict = dict(self.named_parameters())
417 - for name, loaded_weight in hf_model_weights_iterator(
418 - model_name_or_path,
419 - cache_dir,
420 - load_format,
421 - revision,
422 - fall_back_to_pt=False):
414 + for name, loaded_weight in weights:
415 if "rotary_emb.inv_freq" in name:
416 continue
417
vllm/model_executor/models/mixtral_quant.py
+5 -14
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Mixtral model."""
24 -from typing import List, Optional
24 +from typing import Iterable, List, Optional, Tuple
25
26 import numpy as np
27 import torch
@@ -43,9 +43,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 ParallelLMHead, VocabParallelEmbedding)
46 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 -from vllm.model_executor.weight_utils import (default_weight_loader,
48 - hf_model_weights_iterator)
48 from vllm.sequence import SamplerOutput
49
50
@@ -327,6 +326,7 @@ class MixtralModel(nn.Module):
326
327
328 class MixtralForCausalLM(nn.Module):
329 + fall_back_to_pt_during_load = False
330
331 def __init__(
332 self,
@@ -366,11 +366,7 @@ class MixtralForCausalLM(nn.Module):
366 next_tokens = self.sampler(logits, sampling_metadata)
367 return next_tokens
368
369 - def load_weights(self,
370 - model_name_or_path: str,
371 - cache_dir: Optional[str] = None,
372 - load_format: str = "auto",
373 - revision: Optional[str] = None):
369 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
370 stacked_params_mapping = [
371 # (param_name, shard_name, shard_id)
372 ("qkv_proj", "q_proj", "q"),
@@ -379,12 +375,7 @@ class MixtralForCausalLM(nn.Module):
375 ]
376
377 params_dict = dict(self.named_parameters())
382 - for name, loaded_weight in hf_model_weights_iterator(
383 - model_name_or_path,
384 - cache_dir,
385 - load_format,
386 - revision,
387 - fall_back_to_pt=False):
378 + for name, loaded_weight in weights:
379 if "rotary_emb.inv_freq" in name:
380 continue
381 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/mpt.py
+4 -10
@@ -1,7 +1,7 @@
1 # coding=utf-8
2 # Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
3 import math
4 -from typing import List, Optional
4 +from typing import Iterable, List, Optional, Tuple
5
6 import torch
7 import torch.nn as nn
@@ -18,9 +18,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 VocabParallelEmbedding)
21 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 -from vllm.model_executor.weight_utils import (default_weight_loader,
23 - hf_model_weights_iterator)
23 from vllm.sequence import SamplerOutput
24 from vllm.transformers_utils.configs.mpt import MPTConfig
25
@@ -284,14 +283,9 @@ class MPTForCausalLM(nn.Module):
283 next_tokens = self.sampler(logits, sampling_metadata)
284 return next_tokens
285
287 - def load_weights(self,
288 - model_name_or_path: str,
289 - cache_dir: Optional[str] = None,
290 - load_format: str = "auto",
291 - revision: Optional[str] = None):
286 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
287 params_dict = dict(self.named_parameters(remove_duplicate=False))
293 - for name, loaded_weight in hf_model_weights_iterator(
294 - model_name_or_path, cache_dir, load_format, revision):
288 + for name, loaded_weight in weights:
289 # Skip loading extra bias for GPTQ models.
290 if name.endswith(".bias") and name not in params_dict:
291 continue
vllm/model_executor/models/olmo.py
+4 -12
@@ -36,7 +36,7 @@
36 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 """Inference-only OLMo model compatible with HuggingFace weights."""
39 -from typing import List, Optional, Tuple
39 +from typing import Iterable, List, Optional, Tuple
40
41 import torch
42 # this model must need this dependency
@@ -56,9 +56,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
56 from vllm.model_executor.layers.sampler import Sampler
57 from vllm.model_executor.layers.vocab_parallel_embedding import (
58 VocabParallelEmbedding)
59 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
60 from vllm.model_executor.sampling_metadata import SamplingMetadata
60 -from vllm.model_executor.weight_utils import (default_weight_loader,
61 - hf_model_weights_iterator)
61 from vllm.sequence import SamplerOutput
62
63
@@ -348,16 +347,9 @@ class OLMoForCausalLM(nn.Module):
347 next_tokens = self.sampler(logits, sampling_metadata)
348 return next_tokens
349
351 - def load_weights(
352 - self,
353 - model_name_or_path: str,
354 - cache_dir: Optional[str] = None,
355 - load_format: str = "auto",
356 - revision: Optional[str] = None,
357 - ):
350 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
351 params_dict = dict(self.named_parameters(remove_duplicate=False))
359 - for name, loaded_weight in hf_model_weights_iterator(
360 - model_name_or_path, cache_dir, load_format, revision):
352 + for name, loaded_weight in weights:
353 # attention
354 if ".att" in name:
355 name = name.replace(".att", ".attn.att")
vllm/model_executor/models/opt.py
+4 -10
@@ -17,7 +17,7 @@
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
19 """Inference-only OPT model compatible with HuggingFace weights."""
20 -from typing import List, Optional
20 +from typing import Iterable, List, Optional, Tuple
21
22 import torch
23 from torch import nn
@@ -35,9 +35,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
38 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 -from vllm.model_executor.weight_utils import (default_weight_loader,
40 - hf_model_weights_iterator)
40 from vllm.sequence import SamplerOutput
41
42
@@ -315,11 +314,7 @@ class OPTForCausalLM(nn.Module):
314 next_tokens = self.sampler(logits, sampling_metadata)
315 return next_tokens
316
318 - def load_weights(self,
319 - model_name_or_path: str,
320 - cache_dir: Optional[str] = None,
321 - load_format: str = "auto",
322 - revision: Optional[str] = None):
317 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
318 stacked_params_mapping = [
319 # (param_name, shard_name, shard_id)
320 ("qkv_proj", "q_proj", "q"),
@@ -327,8 +322,7 @@ class OPTForCausalLM(nn.Module):
322 ("qkv_proj", "v_proj", "v"),
323 ]
324 params_dict = dict(self.named_parameters(remove_duplicate=False))
330 - for name, loaded_weight in hf_model_weights_iterator(
331 - model_name_or_path, cache_dir, load_format, revision):
325 + for name, loaded_weight in weights:
326 if "lm_head.weight" in name:
327 continue
328 if name.startswith("decoder."):
vllm/model_executor/models/orion.py
+4 -10
@@ -4,7 +4,7 @@
4 # Copyright (c) OrionStar Inc.
5 # LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE
6 """Inference-only Orion-14B model compatible with HuggingFace weights."""
7 -from typing import Any, Dict, List, Optional, Tuple
7 +from typing import Any, Dict, Iterable, List, Optional, Tuple
8
9 import torch
10 from torch import nn
@@ -22,9 +22,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22 from vllm.model_executor.layers.sampler import Sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 ParallelLMHead, VocabParallelEmbedding)
25 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 -from vllm.model_executor.weight_utils import (default_weight_loader,
27 - hf_model_weights_iterator)
27 from vllm.sequence import SamplerOutput
28
29
@@ -280,11 +279,7 @@ class OrionForCausalLM(nn.Module):
279 next_tokens = self.sampler(logits, sampling_metadata)
280 return next_tokens
281
283 - def load_weights(self,
284 - model_name_or_path: str,
285 - cache_dir: Optional[str] = None,
286 - load_format: str = "auto",
287 - revision: Optional[str] = None):
282 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
283 stacked_params_mapping = [
284 # (param_name, shard_name, shard_id)
285 ("qkv_proj", "q_proj", "q"),
@@ -294,8 +289,7 @@ class OrionForCausalLM(nn.Module):
289 ("gate_up_proj", "up_proj", 1),
290 ]
291 params_dict = dict(self.named_parameters())
297 - for name, loaded_weight in hf_model_weights_iterator(
298 - model_name_or_path, cache_dir, load_format, revision):
292 + for name, loaded_weight in weights:
293 if "rotary_emb.inv_freq" in name:
294 continue
295 if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/phi.py
+4 -10
@@ -35,7 +35,7 @@
35 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 """Inference-only Phi-1.5 model compatible with HuggingFace weights."""
38 -from typing import List, Optional
38 +from typing import Iterable, List, Optional, Tuple
39
40 import torch
41 from torch import nn
@@ -53,9 +53,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
53 from vllm.model_executor.layers.sampler import Sampler
54 from vllm.model_executor.layers.vocab_parallel_embedding import (
55 ParallelLMHead, VocabParallelEmbedding)
56 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
57 from vllm.model_executor.sampling_metadata import SamplingMetadata
57 -from vllm.model_executor.weight_utils import (default_weight_loader,
58 - hf_model_weights_iterator)
58 from vllm.sequence import SamplerOutput
59
60
@@ -265,11 +264,7 @@ class PhiForCausalLM(nn.Module):
264 next_tokens = self.sampler(logits, sampling_metadata)
265 return next_tokens
266
268 - def load_weights(self,
269 - model_name_or_path: str,
270 - cache_dir: Optional[str] = None,
271 - load_format: str = "auto",
272 - revision: Optional[str] = None):
267 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
268 stacked_params_mapping = [
269 # (param_name, shard_name, shard_id)
270 ("qkv_proj", "q_proj", "q"),
@@ -278,8 +273,7 @@ class PhiForCausalLM(nn.Module):
273 ]
274 params_dict = dict(self.named_parameters())
275
281 - for name, loaded_weight in hf_model_weights_iterator(
282 - model_name_or_path, cache_dir, load_format, revision):
276 + for name, loaded_weight in weights:
277 if "rotary_emb.inv_freq" in name:
278 continue
279
vllm/model_executor/models/qwen.py
+4 -10
@@ -4,7 +4,7 @@
4 # Copyright (c) Alibaba Cloud.
5 # LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6 """Inference-only QWen model compatible with HuggingFace weights."""
7 -from typing import Any, Dict, List, Optional, Tuple
7 +from typing import Any, Dict, Iterable, List, Optional, Tuple
8
9 import torch
10 from torch import nn
@@ -23,9 +23,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
23 from vllm.model_executor.layers.sampler import Sampler
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 ParallelLMHead, VocabParallelEmbedding)
26 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 -from vllm.model_executor.weight_utils import (default_weight_loader,
28 - hf_model_weights_iterator)
28 from vllm.sequence import SamplerOutput
29
30
@@ -253,19 +252,14 @@ class QWenLMHeadModel(nn.Module):
252 next_tokens = self.sampler(logits, sampling_metadata)
253 return next_tokens
254
256 - def load_weights(self,
257 - model_name_or_path: str,
258 - cache_dir: Optional[str] = None,
259 - load_format: str = "auto",
260 - revision: Optional[str] = None):
255 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
256 stacked_params_mapping = [
257 # (param_name, shard_name, shard_id)
258 ("gate_up_proj", "w2", 0),
259 ("gate_up_proj", "w1", 1),
260 ]
261 params_dict = dict(self.named_parameters())
267 - for name, loaded_weight in hf_model_weights_iterator(
268 - model_name_or_path, cache_dir, load_format, revision):
262 + for name, loaded_weight in weights:
263 if "rotary_emb.inv_freq" in name:
264 continue
265 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/qwen2.py
+4 -10
@@ -22,7 +22,7 @@
22 # See the License for the specific language governing permissions and
23 # limitations under the License.
24 """Inference-only Qwen2 model compatible with HuggingFace weights."""
25 -from typing import List, Optional, Tuple
25 +from typing import Iterable, List, Optional, Tuple
26
27 import torch
28 from torch import nn
@@ -42,9 +42,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 ParallelLMHead, VocabParallelEmbedding)
45 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.model_executor.weight_utils import (default_weight_loader,
47 - hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
48
49
@@ -331,11 +330,7 @@ class Qwen2ForCausalLM(nn.Module):
330 next_tokens = self.sampler(logits, sampling_metadata)
331 return next_tokens
332
334 - def load_weights(self,
335 - model_name_or_path: str,
336 - cache_dir: Optional[str] = None,
337 - load_format: str = "auto",
338 - revision: Optional[str] = None):
333 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
334 stacked_params_mapping = [
335 # (param_name, shard_name, shard_id)
336 ("qkv_proj", "q_proj", "q"),
@@ -345,8 +340,7 @@ class Qwen2ForCausalLM(nn.Module):
340 ("gate_up_proj", "up_proj", 1),
341 ]
342 params_dict = dict(self.named_parameters(remove_duplicate=False))
348 - for name, loaded_weight in hf_model_weights_iterator(
349 - model_name_or_path, cache_dir, load_format, revision):
343 + for name, loaded_weight in weights:
344 if "rotary_emb.inv_freq" in name:
345 continue
346 if self.config.tie_word_embeddings and "lm_head.weight" in name:
vllm/model_executor/models/qwen2_moe.py
+6 -14
@@ -22,7 +22,7 @@
22 # See the License for the specific language governing permissions and
23 # limitations under the License.
24 """Inference-only Qwen2MoE model compatible with HuggingFace weights."""
25 -from typing import Any, Dict, List, Optional
25 +from typing import Any, Dict, Iterable, List, Optional, Tuple
26
27 import torch
28 import torch.nn.functional as F
@@ -46,9 +46,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
46 from vllm.model_executor.layers.sampler import Sampler
47 from vllm.model_executor.layers.vocab_parallel_embedding import (
48 ParallelLMHead, VocabParallelEmbedding)
49 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 -from vllm.model_executor.weight_utils import (default_weight_loader,
51 - hf_model_weights_iterator)
51 from vllm.sequence import SamplerOutput
52
53
@@ -366,6 +365,8 @@ class Qwen2MoeModel(nn.Module):
365
366 class Qwen2MoeForCausalLM(nn.Module):
367
368 + fall_back_to_pt_during_load = False
369 +
370 def __init__(
371 self,
372 config: PretrainedConfig,
@@ -404,11 +405,7 @@ class Qwen2MoeForCausalLM(nn.Module):
405 next_tokens = self.sampler(logits, sampling_metadata)
406 return next_tokens
407
407 - def load_weights(self,
408 - model_name_or_path: str,
409 - cache_dir: Optional[str] = None,
410 - load_format: str = "auto",
411 - revision: Optional[str] = None):
408 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
409 stacked_params_mapping = [
410 # (param_name, shard_name, shard_id)
411 ("qkv_proj", "q_proj", "q"),
@@ -419,12 +416,7 @@ class Qwen2MoeForCausalLM(nn.Module):
416 ]
417
418 params_dict = dict(self.named_parameters())
422 - for name, loaded_weight in hf_model_weights_iterator(
423 - model_name_or_path,
424 - cache_dir,
425 - load_format,
426 - revision,
427 - fall_back_to_pt=False):
419 + for name, loaded_weight in weights:
420 if "rotary_emb.inv_freq" in name:
421 continue
422 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/stablelm.py
+4 -10
@@ -19,7 +19,7 @@
19 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
20 """Inference-only StabeLM (https://github.com/Stability-AI/StableLM)
21 model compatible with HuggingFace weights."""
22 -from typing import List, Optional, Tuple
22 +from typing import Iterable, List, Optional, Tuple
23
24 import torch
25 from torch import nn
@@ -37,9 +37,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 -from vllm.model_executor.weight_utils import (default_weight_loader,
42 - hf_model_weights_iterator)
42 from vllm.sequence import SamplerOutput
43
44
@@ -262,11 +261,7 @@ class StablelmForCausalLM(nn.Module):
261 next_tokens = self.sampler(logits, sampling_metadata)
262 return next_tokens
263
265 - def load_weights(self,
266 - model_name_or_path: str,
267 - cache_dir: Optional[str] = None,
268 - load_format: str = "auto",
269 - revision: Optional[str] = None):
264 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
265 stacked_params_mapping = [
266 # (param_name, shard_name, shard_id)
267 ("qkv_proj", "q_proj", "q"),
@@ -276,8 +271,7 @@ class StablelmForCausalLM(nn.Module):
271 ("gate_up_proj", "up_proj", 1),
272 ]
273 params_dict = dict(self.named_parameters())
279 - for name, loaded_weight in hf_model_weights_iterator(
280 - model_name_or_path, cache_dir, load_format, revision):
274 + for name, loaded_weight in weights:
275 if "rotary_emb.inv_freq" in name:
276 continue
277 if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/starcoder2.py
+4 -10
@@ -18,7 +18,7 @@
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 """ PyTorch Starcoder2 model."""
21 -from typing import List, Optional
21 +from typing import Iterable, List, Optional, Tuple
22
23 import torch
24 from torch import nn
@@ -36,9 +36,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
39 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 -from vllm.model_executor.weight_utils import (default_weight_loader,
41 - hf_model_weights_iterator)
41 from vllm.sequence import SamplerOutput
42
43
@@ -274,11 +273,7 @@ class Starcoder2ForCausalLM(nn.Module):
273 next_tokens = self.sampler(logits, sampling_metadata)
274 return next_tokens
275
277 - def load_weights(self,
278 - model_name_or_path: str,
279 - cache_dir: Optional[str] = None,
280 - load_format: str = "auto",
281 - revision: Optional[str] = None):
276 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
277 stacked_params_mapping = [
278 # (param_name, shard_name, shard_id)
279 ("qkv_proj", "q_proj", "q"),
@@ -287,8 +282,7 @@ class Starcoder2ForCausalLM(nn.Module):
282 ]
283
284 params_dict = dict(self.named_parameters(remove_duplicate=False))
290 - for name, loaded_weight in hf_model_weights_iterator(
291 - model_name_or_path, cache_dir, load_format, revision):
285 + for name, loaded_weight in weights:
286 if "rotary_emb.inv_freq" in name:
287 continue
288
vllm/model_executor/models/xverse.py
+4 -10
@@ -20,7 +20,7 @@
20 # See the License for the specific language governing permissions and
21 # limitations under the License.
22 """Inference-only Xverse model compatible with HuggingFace weights."""
23 -from typing import Any, Dict, List, Optional, Tuple
23 +from typing import Any, Dict, Iterable, List, Optional, Tuple
24
25 import torch
26 from torch import nn
@@ -40,9 +40,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
43 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 -from vllm.model_executor.weight_utils import (default_weight_loader,
45 - hf_model_weights_iterator)
45 from vllm.sequence import SamplerOutput
46
47
@@ -331,11 +330,7 @@ class XverseForCausalLM(nn.Module):
330 next_tokens = self.sampler(logits, sampling_metadata)
331 return next_tokens
332
334 - def load_weights(self,
335 - model_name_or_path: str,
336 - cache_dir: Optional[str] = None,
337 - load_format: str = "auto",
338 - revision: Optional[str] = None):
333 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
334 stacked_params_mapping = [
335 ("qkv_proj", "q_proj", "q"),
336 ("qkv_proj", "k_proj", "k"),
@@ -344,8 +339,7 @@ class XverseForCausalLM(nn.Module):
339 ("gate_up_proj", "up_proj", 1),
340 ]
341 params_dict = dict(self.named_parameters())
347 - for name, loaded_weight in hf_model_weights_iterator(
348 - model_name_or_path, cache_dir, load_format, revision):
342 + for name, loaded_weight in weights:
343 if ("rotary_emb.inv_freq" in name
344 or "rotary_emb.cos_cached" in name
345 or "rotary_emb.sin_cached" in name):
vllm/transformers_utils/tokenizer.py
+20 -1
@@ -1,8 +1,10 @@
1 +import os
2 from typing import Optional, Union
3
4 from transformers import (AutoTokenizer, PreTrainedTokenizer,
5 PreTrainedTokenizerFast)
6
7 +from vllm.config import VLLM_USE_MODELSCOPE
8 from vllm.logger import init_logger
9 from vllm.lora.request import LoRARequest
10 from vllm.transformers_utils.tokenizers import BaichuanTokenizer
@@ -57,9 +59,26 @@ def get_tokenizer(
59 tokenizer_mode: str = "auto",
60 trust_remote_code: bool = False,
61 tokenizer_revision: Optional[str] = None,
62 + download_dir: Optional[str] = None,
63 **kwargs,
64 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
62 - """Gets a tokenizer for the given model name via Huggingface."""
65 + """Gets a tokenizer for the given model name via Huggingface/modelscope."""
66 + if VLLM_USE_MODELSCOPE:
67 + # download model from ModelScope hub,
68 + # lazy import so that modelscope is not required for normal use.
69 + # pylint: disable=C.
70 + from modelscope.hub.snapshot_download import snapshot_download
71 +
72 + # Only set the tokenizer here, model will be downloaded on the workers.
73 + if not os.path.exists(tokenizer_name):
74 + tokenizer_path = snapshot_download(
75 + model_id=tokenizer_name,
76 + cache_dir=download_dir,
77 + revision=tokenizer_revision,
78 + # Ignore weights - we only need the tokenizer.
79 + ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
80 + tokenizer_name = tokenizer_path
81 +
82 if tokenizer_mode == "slow":
83 if kwargs.get("use_fast", False):
84 raise ValueError(
vllm/worker/cpu_model_runner.py
+8 -4
@@ -3,8 +3,8 @@ from typing import Dict, List, Optional, Tuple
3 import torch
4
5 from vllm.attention import AttentionMetadata, get_attn_backend
6 -from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7 - SchedulerConfig)
6 +from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
7 + ParallelConfig, SchedulerConfig)
8 from vllm.distributed import broadcast_tensor_dict
9 from vllm.logger import init_logger
10 from vllm.model_executor import SamplingMetadata
@@ -26,6 +26,7 @@ class CPUModelRunner:
26 parallel_config: ParallelConfig,
27 scheduler_config: SchedulerConfig,
28 device_config: DeviceConfig,
29 + load_config: LoadConfig,
30 lora_config: Optional[LoRAConfig],
31 kv_cache_dtype: Optional[str] = "auto",
32 is_driver_worker: bool = False,
@@ -36,6 +37,7 @@ class CPUModelRunner:
37 self.parallel_config = parallel_config
38 self.scheduler_config = scheduler_config
39 self.lora_config = lora_config
40 + self.load_config = load_config
41 self.is_driver_worker = is_driver_worker
42
43 # model_config can be None in tests/samplers/test_sampler.py.
@@ -55,8 +57,10 @@ class CPUModelRunner:
57 self.model_config.dtype if model_config is not None else None)
58
59 def load_model(self) -> None:
58 - self.model = get_model(self.model_config,
59 - self.device_config,
60 + self.model = get_model(model_config=self.model_config,
61 + load_config=self.load_config,
62 + device_config=self.device_config,
63 + vision_language_config=None,
64 lora_config=self.lora_config,
65 parallel_config=self.parallel_config,
66 scheduler_config=self.scheduler_config)
vllm/worker/cpu_worker.py
+5 -2
@@ -5,8 +5,8 @@ import torch
5 import torch.distributed
6
7 from vllm.attention import get_attn_backend
8 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 - ParallelConfig, SchedulerConfig)
8 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 + ModelConfig, ParallelConfig, SchedulerConfig)
10 from vllm.distributed import (broadcast_tensor_dict,
11 ensure_model_parallel_initialized,
12 init_distributed_environment)
@@ -117,6 +117,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
117 scheduler_config: SchedulerConfig,
118 device_config: DeviceConfig,
119 cache_config: CacheConfig,
120 + load_config: LoadConfig,
121 local_rank: int,
122 rank: int,
123 distributed_init_method: str,
@@ -129,6 +130,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
130 self.scheduler_config = scheduler_config
131 self.device_config = device_config
132 self.cache_config = cache_config
133 + self.load_config = load_config
134 self.local_rank = local_rank
135 self.rank = rank
136 self.distributed_init_method = distributed_init_method
@@ -141,6 +143,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
143 parallel_config,
144 scheduler_config,
145 device_config,
146 + load_config=self.load_config,
147 lora_config=self.lora_config,
148 kv_cache_dtype=kv_cache_dtype,
149 is_driver_worker=is_driver_worker)
vllm/worker/model_runner.py
+7 -8
@@ -9,9 +9,8 @@ import torch.nn as nn
9
10 from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11 get_attn_backend)
12 -from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
13 - SchedulerConfig, TensorizerConfig,
14 - VisionLanguageConfig)
12 +from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
13 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
14 from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
15 from vllm.distributed.device_communicators import (custom_all_reduce,
16 pynccl_utils)
@@ -108,17 +107,17 @@ class ModelRunner:
107 parallel_config: ParallelConfig,
108 scheduler_config: SchedulerConfig,
109 device_config: DeviceConfig,
110 + load_config: LoadConfig,
111 lora_config: Optional[LoRAConfig],
112 kv_cache_dtype: Optional[str] = "auto",
113 is_driver_worker: bool = False,
114 vision_language_config: Optional[VisionLanguageConfig] = None,
115 - tensorizer_config: Optional[TensorizerConfig] = None,
115 ):
116 self.model_config = model_config
117 self.parallel_config = parallel_config
118 self.scheduler_config = scheduler_config
119 self.lora_config = lora_config
121 - self.tensorizer_config = tensorizer_config
120 + self.load_config = load_config
121 self.is_driver_worker = is_driver_worker
122
123 # model_config can be None in tests/samplers/test_sampler.py.
@@ -156,13 +155,13 @@ class ModelRunner:
155 def load_model(self) -> None:
156 with CudaMemoryProfiler() as m:
157 self.model = get_model(
159 - self.model_config,
160 - self.device_config,
158 + model_config=self.model_config,
159 + device_config=self.device_config,
160 + load_config=self.load_config,
161 lora_config=self.lora_config,
162 vision_language_config=self.vision_language_config,
163 parallel_config=self.parallel_config,
164 scheduler_config=self.scheduler_config,
165 - tensorizer_config=self.tensorizer_config,
165 )
166
167 self.model_memory_usage = m.consumed_memory
vllm/worker/neuron_model_runner.py
+1 -1
@@ -6,7 +6,7 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
6 SchedulerConfig)
7 from vllm.logger import init_logger
8 from vllm.model_executor import SamplingMetadata
9 -from vllm.model_executor.neuron_model_loader import get_neuron_model
9 +from vllm.model_executor.model_loader.neuron import get_neuron_model
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
12 from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
vllm/worker/worker.py
+5 -5
@@ -6,8 +6,8 @@ from typing import Dict, List, Optional, Set, Tuple
6 import torch
7 import torch.distributed
8
9 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10 - ParallelConfig, SchedulerConfig, TensorizerConfig,
9 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 + ModelConfig, ParallelConfig, SchedulerConfig,
11 VisionLanguageConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
@@ -38,12 +38,12 @@ class Worker(WorkerBase):
38 scheduler_config: SchedulerConfig,
39 device_config: DeviceConfig,
40 cache_config: CacheConfig,
41 + load_config: LoadConfig,
42 local_rank: int,
43 rank: int,
44 distributed_init_method: str,
45 lora_config: Optional[LoRAConfig] = None,
46 vision_language_config: Optional[VisionLanguageConfig] = None,
46 - tensorizer_config: Optional[TensorizerConfig] = None,
47 is_driver_worker: bool = False,
48 ) -> None:
49 self.model_config = model_config
@@ -55,7 +55,7 @@ class Worker(WorkerBase):
55 self.rank = rank
56 self.distributed_init_method = distributed_init_method
57 self.lora_config = lora_config
58 - self.tensorizer_config = tensorizer_config
58 + self.load_config = load_config
59 self.is_driver_worker = is_driver_worker
60 if self.is_driver_worker:
61 assert self.rank == 0, "The driver worker must have rank 0."
@@ -70,11 +70,11 @@ class Worker(WorkerBase):
70 parallel_config,
71 scheduler_config,
72 device_config,
73 + load_config=load_config,
74 lora_config=self.lora_config,
75 kv_cache_dtype=self.cache_config.cache_dtype,
76 is_driver_worker=is_driver_worker,
77 vision_language_config=vision_language_config,
77 - tensorizer_config=tensorizer_config,
78 )
79 # Uninitialized cache engine. Will be initialized by
80 # initialize_cache.