A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Refactor model loading code (#4097)
Antoni Baum committed
Apr 16, 2024 at 11:34 UTC
69e1d2fb6922b2d388bae41286d8867976cbd6c6
67 files changed
+1064
-973
.buildkite/test-pipeline.yaml
+1
-1
@@ -92,7 +92,7 @@ steps:
92
parallelism: 4
93
94
- label: Tensorizer Test
95
- command: apt-get install curl libsodium23 && pytest -v -s tensorizer
95
+ command: apt-get install curl libsodium23 && pytest -v -s tensorizer_loader
96
97
- label: Metrics Test
98
command: pytest -v -s metrics
examples/fp8/extract_scales.py
+2
-2
@@ -11,7 +11,7 @@ from safetensors.torch import safe_open
11
from vllm.model_executor.layers.quantization.schema import QuantParamSchema
12
13
14
-# Adapted from vllm/model_executor/weight_utils.py
14
+# Adapted from vllm/model_executor/model_loader/weight_utils.py
15
# The main differences are that we add the NPZ format and simplify
16
# its functionality drastically for our purposes (e.g. we assume that
17
# the quantized model exists locally and there is no need to download it)
@@ -71,7 +71,7 @@ def _prepare_hf_weights(
71
return hf_weights_files, use_safetensors
72
73
74
-# Adapted from vllm/model_executor/weight_utils.py
74
+# Adapted from vllm/model_executor/model_loader/weight_utils.py
75
def _hf_tensorfile_iterator(filename: str, load_format: str,
76
use_safetensors: bool):
77
if load_format == "npz":
examples/tensorize_vllm_model.py
+1
-1
@@ -16,8 +16,8 @@ from transformers import AutoConfig, PretrainedConfig
16
from vllm.distributed import initialize_model_parallel
17
from vllm.engine.arg_utils import EngineArgs
18
from vllm.engine.llm_engine import LLMEngine
19
+from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20
from vllm.model_executor.models import ModelRegistry
20
-from vllm.model_executor.tensorizer_loader import TensorizerArgs
21
22
# yapf conflicts with isort for this docstring
23
# yapf: disable
tests/lora/conftest.py
+5
-5
@@ -153,11 +153,11 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
153
cleanup()
154
get_model_old = get_model
155
156
- def get_model_patched(model_config, device_config, **kwargs):
157
- return get_model_old(model_config,
158
- device_config,
159
- lora_config=LoRAConfig(max_loras=4,
160
- max_lora_rank=8))
156
+ def get_model_patched(*, model_config, device_config, **kwargs):
157
+ kwargs["lora_config"] = LoRAConfig(max_loras=4, max_lora_rank=8)
158
+ return get_model_old(model_config=model_config,
159
+ device_config=device_config,
160
+ **kwargs)
161
162
with patch("vllm.worker.model_runner.get_model", get_model_patched):
163
engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
tests/lora/test_worker.py
+6
-4
@@ -3,8 +3,8 @@ import random
3
import tempfile
4
from unittest.mock import patch
5
6
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7
- ParallelConfig, SchedulerConfig)
6
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
+ ModelConfig, ParallelConfig, SchedulerConfig)
8
from vllm.lora.models import LoRAMapping
9
from vllm.lora.request import LoRARequest
10
from vllm.worker.worker import Worker
@@ -18,12 +18,14 @@ def test_worker_apply_lora(sql_lora_files):
18
"meta-llama/Llama-2-7b-hf",
19
tokenizer_mode="auto",
20
trust_remote_code=False,
21
- download_dir=None,
22
- load_format="dummy",
21
seed=0,
22
dtype="float16",
23
revision=None,
24
),
25
+ load_config=LoadConfig(
26
+ download_dir=None,
27
+ load_format="dummy",
28
+ ),
29
parallel_config=ParallelConfig(1, 1, False),
30
scheduler_config=SchedulerConfig(32, 32, 32),
31
device_config=DeviceConfig("cuda"),
tests/model_executor/weight_utils.py
+1
-1
@@ -3,7 +3,7 @@ import os
3
import huggingface_hub.constants
4
import pytest
5
6
-from vllm.model_executor.weight_utils import enable_hf_transfer
6
+from vllm.model_executor.model_loader.weight_utils import enable_hf_transfer
7
8
9
def test_hf_transfer_auto_activation():
tests/quantization/test_autogptq_marlin_configs.py
-4
@@ -36,8 +36,6 @@ def test_auto_gptq(model_quant_type: str, ) -> None:
36
model_path,
37
tokenizer_mode="auto",
38
trust_remote_code=False,
39
- download_dir=None,
40
- load_format="dummy",
39
seed=0,
40
dtype="float16",
41
revision=None,
@@ -49,8 +47,6 @@ def test_auto_gptq(model_quant_type: str, ) -> None:
47
model_path,
48
tokenizer_mode="auto",
49
trust_remote_code=False,
52
- download_dir=None,
53
- load_format="dummy",
50
seed=0,
51
dtype="float16",
52
revision=None,
tests/samplers/test_sampler.py
+12
-2
@@ -32,7 +32,12 @@ def _prepare_test(
32
1e-2,
33
dtype=input_tensor.dtype)
34
sampler = MockLogitsSampler(fake_logits)
35
- model_runner = ModelRunner(None, None, None, None, None)
35
+ model_runner = ModelRunner(model_config=None,
36
+ parallel_config=None,
37
+ scheduler_config=None,
38
+ device_config=None,
39
+ load_config=None,
40
+ lora_config=None)
41
return input_tensor, fake_logits, sampler, model_runner
42
43
@@ -591,7 +596,12 @@ def test_sampler_top_k_top_p(seed: int, device: str):
596
device=input_tensor.device,
597
dtype=input_tensor.dtype)
598
sampler = MockLogitsSampler(fake_logits)
594
- model_runner = ModelRunner(None, None, None, None, None)
599
+ model_runner = ModelRunner(model_config=None,
600
+ parallel_config=None,
601
+ scheduler_config=None,
602
+ device_config=None,
603
+ load_config=None,
604
+ lora_config=None)
605
606
generation_model = GenerationMixin()
607
generation_config = GenerationConfig(top_k=top_k,
tests/spec_decode/utils.py
+1
@@ -118,6 +118,7 @@ def create_worker(cls: type,
118
scheduler_config=engine_config.scheduler_config,
119
device_config=engine_config.device_config,
120
cache_config=engine_config.cache_config,
121
+ load_config=engine_config.load_config,
122
local_rank=0,
123
rank=0,
124
distributed_init_method=distributed_init_method,
tests/tensorizer_loader/__init__.py
renamed
tests/tensorizer_loader/tensorize_vllm_model_for_testing.py
renamed
+2
-2
@@ -16,8 +16,8 @@ from transformers import AutoConfig, PretrainedConfig
16
from vllm.distributed import initialize_model_parallel
17
from vllm.engine.arg_utils import EngineArgs
18
from vllm.engine.llm_engine import LLMEngine
19
+from vllm.model_executor.model_loader.tensorizer import TensorizerArgs
20
from vllm.model_executor.models import ModelRegistry
20
-from vllm.model_executor.tensorizer_loader import TensorizerArgs
21
22
# yapf conflicts with isort for this docstring
23
# yapf: disable
@@ -74,7 +74,7 @@ def parse_args():
74
"extremely quickly. Tensor encryption and decryption is "
75
"also supported, although libsodium must be installed to "
76
"use it.")
77
- parser = EngineArgs.add_cli_args(parser)
77
+ parser = TensorizerArgs.add_cli_args(EngineArgs.add_cli_args(parser))
78
subparsers = parser.add_subparsers(dest='command')
79
80
serialize_parser = subparsers.add_parser(
tests/tensorizer_loader/test_tensorizer.py
renamed
+82
-57
@@ -1,16 +1,19 @@
1
import gc
2
+import json
3
+import os
4
import subprocess
5
from unittest.mock import MagicMock, patch
6
7
+import openai
8
import pytest
9
+import ray
10
import torch
11
12
from tests.entrypoints.test_openai_server import ServerRunner
13
from vllm import SamplingParams
10
-from vllm.config import TensorizerConfig
11
-from vllm.model_executor.tensorizer_loader import (
12
- EncryptionParams, TensorSerializer, is_vllm_serialized_tensorizer,
13
- load_with_tensorizer, open_stream)
14
+from vllm.model_executor.model_loader.tensorizer import (
15
+ EncryptionParams, TensorizerConfig, TensorSerializer,
16
+ is_vllm_serialized_tensorizer, load_with_tensorizer, open_stream)
17
18
prompts = [
19
"Hello, my name is",
@@ -22,6 +25,8 @@ prompts = [
25
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
26
27
model_ref = "facebook/opt-125m"
28
+tensorize_model_for_testing_script = os.path.join(
29
+ os.path.dirname(__file__), "tensorize_vllm_model_for_testing.py")
30
31
32
def is_curl_installed():
@@ -38,7 +43,7 @@ def tensorizer_config():
43
return config
44
45
41
-@patch('vllm.model_executor.tensorizer_loader.TensorizerAgent')
46
+@patch('vllm.model_executor.model_loader.tensorizer.TensorizerAgent')
47
def test_load_with_tensorizer(mock_agent, tensorizer_config):
48
mock_linear_method = MagicMock()
49
mock_agent_instance = mock_agent.return_value
@@ -81,11 +86,13 @@ def test_deserialized_vllm_model_has_same_outputs(vllm_runner, tmp_path):
86
del vllm_model, model
87
gc.collect()
88
torch.cuda.empty_cache()
84
- loaded_vllm_model = vllm_runner(model_ref,
85
- load_format="tensorizer",
86
- tensorizer_uri=model_path,
87
- num_readers=1,
88
- vllm_tensorized=True)
89
+ loaded_vllm_model = vllm_runner(
90
+ model_ref,
91
+ load_format="tensorizer",
92
+ model_loader_extra_config=TensorizerConfig(tensorizer_uri=model_path,
93
+ num_readers=1,
94
+ vllm_tensorized=True),
95
+ )
96
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
97
98
# Assumes SamplingParams being seeded ensures the outputs are deterministic
@@ -97,14 +104,14 @@ def test_can_deserialize_s3(vllm_runner):
104
model_ref = "EleutherAI/pythia-1.4b"
105
tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
106
100
- loaded_hf_model = vllm_runner(
101
- model_ref,
102
- tensorizer_uri=tensorized_path,
103
- load_format="tensorizer",
104
- num_readers=1,
105
- vllm_tensorized=False,
106
- s3_endpoint="object.ord1.coreweave.com",
107
- )
107
+ loaded_hf_model = vllm_runner(model_ref,
108
+ load_format="tensorizer",
109
+ model_loader_extra_config=TensorizerConfig(
110
+ tensorizer_uri=tensorized_path,
111
+ num_readers=1,
112
+ vllm_tensorized=False,
113
+ s3_endpoint="object.ord1.coreweave.com",
114
+ ))
115
116
deserialized_outputs = loaded_hf_model.generate(prompts, sampling_params)
117
@@ -131,11 +138,12 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
138
gc.collect()
139
torch.cuda.empty_cache()
140
loaded_vllm_model = vllm_runner(model_ref,
134
- tensorizer_uri=model_path,
141
load_format="tensorizer",
136
- encryption_keyfile=key_path,
137
- num_readers=1,
138
- vllm_tensorized=True)
142
+ model_loader_extra_config=TensorizerConfig(
143
+ tensorizer_uri=model_path,
144
+ encryption_keyfile=key_path,
145
+ num_readers=1,
146
+ vllm_tensorized=True))
147
148
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
149
@@ -156,10 +164,11 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
164
gc.collect()
165
torch.cuda.empty_cache()
166
loaded_hf_model = vllm_runner(model_ref,
159
- tensorizer_uri=model_path,
167
load_format="tensorizer",
161
- num_readers=1,
162
- vllm_tensorized=False)
168
+ model_loader_extra_config=TensorizerConfig(
169
+ tensorizer_uri=model_path,
170
+ num_readers=1,
171
+ vllm_tensorized=False))
172
173
deserialized_outputs = loaded_hf_model.generate_greedy(
174
prompts, max_tokens=max_tokens)
@@ -190,10 +199,12 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
199
torch.cuda.empty_cache()
200
loaded_vllm_model = vllm_runner(
201
model_ref,
193
- tensorizer_uri=model_path,
202
load_format="tensorizer",
195
- num_readers=1,
196
- vllm_tensorized=True,
203
+ model_loader_extra_config=TensorizerConfig(
204
+ tensorizer_uri=model_path,
205
+ num_readers=1,
206
+ vllm_tensorized=True,
207
+ ),
208
enable_lora=True,
209
max_loras=1,
210
max_lora_rank=8,
@@ -208,16 +219,18 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
219
220
def test_load_without_tensorizer_load_format(vllm_runner):
221
with pytest.raises(ValueError):
211
- vllm_runner(model_ref, tensorizer_uri="test")
222
+ vllm_runner(model_ref,
223
+ model_loader_extra_config=TensorizerConfig(
224
+ tensorizer_uri="test", vllm_tensorized=False))
225
226
227
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
228
def test_tensorize_vllm_model(tmp_path):
229
# Test serialize command
230
serialize_args = [
218
- "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
219
- model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
220
- tmp_path, "--suffix", "tests"
231
+ "python3", tensorize_model_for_testing_script, "--model", model_ref,
232
+ "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
233
+ "--suffix", "tests"
234
]
235
result = subprocess.run(serialize_args, capture_output=True, text=True)
236
print(result.stdout) # Print the output of the serialize command
@@ -229,8 +242,8 @@ def test_tensorize_vllm_model(tmp_path):
242
243
# Test deserialize command
244
deserialize_args = [
232
- "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
233
- model_ref, "--dtype", "float16", "deserialize", "--path-to-tensors",
245
+ "python3", tensorize_model_for_testing_script, "--model", model_ref,
246
+ "--dtype", "float16", "deserialize", "--path-to-tensors",
247
path_to_tensors
248
]
249
result = subprocess.run(deserialize_args, capture_output=True, text=True)
@@ -242,9 +255,9 @@ def test_tensorize_vllm_model(tmp_path):
255
def test_openai_apiserver_with_tensorizer(tmp_path):
256
## Serialize model
257
serialize_args = [
245
- "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
246
- model_ref, "--dtype", "float16", "serialize", "--serialized-directory",
247
- tmp_path, "--suffix", "tests"
258
+ "python3", tensorize_model_for_testing_script, "--model", model_ref,
259
+ "--dtype", "float16", "serialize", "--serialized-directory", tmp_path,
260
+ "--suffix", "tests"
261
]
262
result = subprocess.run(serialize_args, capture_output=True, text=True)
263
print(result.stdout) # Print the output of the serialize command
@@ -253,25 +266,47 @@ def test_openai_apiserver_with_tensorizer(tmp_path):
266
f"\n{result.stdout}\n{result.stderr}")
267
268
path_to_tensors = f"{tmp_path}/vllm/{model_ref}/tests/model.tensors"
269
+ model_loader_extra_config = {
270
+ "tensorizer_uri": path_to_tensors,
271
+ "vllm_tensorized": True
272
+ }
273
274
## Start OpenAI API server
275
openai_args = [
276
"--model", model_ref, "--dtype", "float16", "--load-format",
260
- "tensorizer", "--tensorizer-uri", path_to_tensors, "--vllm-tensorized",
261
- "--port", "8000"
277
+ "tensorizer", "--model-loader-extra-config",
278
+ json.dumps(model_loader_extra_config), "--port", "8000"
279
]
280
281
server = ServerRunner.remote(openai_args)
282
283
+ assert ray.get(server.ready.remote())
284
print("Server ready.")
267
- assert server.ready.remote()
285
+
286
+ client = openai.OpenAI(
287
+ base_url="http://localhost:8000/v1",
288
+ api_key="token-abc123",
289
+ )
290
+ completion = client.completions.create(model=model_ref,
291
+ prompt="Hello, my name is",
292
+ max_tokens=5,
293
+ temperature=0.0)
294
+
295
+ assert completion.id is not None
296
+ assert completion.choices is not None and len(completion.choices) == 1
297
+ assert completion.choices[0].text is not None and len(
298
+ completion.choices[0].text) >= 5
299
+ assert completion.choices[0].finish_reason == "length"
300
+ assert completion.usage == openai.types.CompletionUsage(
301
+ completion_tokens=5, prompt_tokens=6, total_tokens=11)
302
303
304
def test_raise_value_error_on_invalid_load_format(vllm_runner):
305
with pytest.raises(ValueError):
306
vllm_runner(model_ref,
307
load_format="safetensors",
274
- tensorizer_uri="test")
308
+ model_loader_extra_config=TensorizerConfig(
309
+ tensorizer_uri="test", vllm_tensorized=False))
310
311
312
def test_tensorizer_with_tp(vllm_runner):
@@ -281,22 +316,12 @@ def test_tensorizer_with_tp(vllm_runner):
316
317
vllm_runner(
318
model_ref,
284
- tensorizer_uri=tensorized_path,
319
load_format="tensorizer",
286
- num_readers=1,
287
- vllm_tensorized=False,
288
- s3_endpoint="object.ord1.coreweave.com",
320
+ model_loader_extra_config=TensorizerConfig(
321
+ tensorizer_uri=tensorized_path,
322
+ num_readers=1,
323
+ vllm_tensorized=False,
324
+ s3_endpoint="object.ord1.coreweave.com",
325
+ ),
326
tensor_parallel_size=2,
327
)
291
-
292
-
293
-@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
294
-def test_tensorizer_warn_quant(tmp_path):
295
- model_ref = "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit"
296
- serialize_args = [
297
- "python3", "tensorizer/tensorize_vllm_model_for_testing.py", "--model",
298
- model_ref, "--quantization", "gptq", "--tensorizer-uri", "test",
299
- "serialize", "--serialized-directory", tmp_path, "--suffix", "tests"
300
- ]
301
- result = subprocess.run(serialize_args, capture_output=True, text=True)
302
- assert 'PerformanceWarning' in result.stderr
tests/test_config.py
-4
@@ -11,8 +11,6 @@ def test_get_sliding_window():
11
"Qwen/Qwen1.5-7B",
12
tokenizer_mode="auto",
13
trust_remote_code=False,
14
- download_dir=None,
15
- load_format="dummy",
14
seed=0,
15
dtype="float16",
16
revision=None,
@@ -30,8 +28,6 @@ def test_get_sliding_window():
28
"mistralai/Mistral-7B-v0.1",
29
tokenizer_mode="auto",
30
trust_remote_code=False,
33
- download_dir=None,
34
- load_format="dummy",
31
seed=0,
32
dtype="float16",
33
revision=None,
tests/test_logits_processor.py
+6
-1
@@ -37,7 +37,12 @@ def _prepare_test(
37
1e-2,
38
dtype=input_tensor.dtype)
39
logits_processor = MockLogitsProcessor(32000, 0.5, fake_logits)
40
- model_runner = ModelRunner(None, None, None, None, None)
40
+ model_runner = ModelRunner(model_config=None,
41
+ parallel_config=None,
42
+ scheduler_config=None,
43
+ device_config=None,
44
+ load_config=None,
45
+ lora_config=None)
46
return input_tensor, fake_logits, logits_processor, model_runner
47
48
tests/worker/test_model_runner.py
+24
-15
@@ -12,7 +12,12 @@ def test_prepare_prompt(batch_size):
12
100000,
13
100000,
14
enable_chunked_prefill=False)
15
- model_runner = ModelRunner(None, None, scheduler_config, None, None)
15
+ model_runner = ModelRunner(model_config=None,
16
+ parallel_config=None,
17
+ scheduler_config=scheduler_config,
18
+ device_config=None,
19
+ load_config=None,
20
+ lora_config=None)
21
model_runner.set_block_size(16)
22
23
prompt_lens = []
@@ -118,8 +123,6 @@ def test_prepare_decode_cuda_graph(batch_size):
123
"facebook/opt-125m",
124
tokenizer_mode="auto",
125
trust_remote_code=False,
121
- download_dir=None,
122
- load_format="dummy",
126
seed=0,
127
dtype="float16",
128
revision=None,
@@ -129,8 +132,12 @@ def test_prepare_decode_cuda_graph(batch_size):
132
100000,
133
100000,
134
enable_chunked_prefill=False)
132
- model_runner = ModelRunner(model_config, None, scheduler_config, None,
133
- None)
135
+ model_runner = ModelRunner(model_config=model_config,
136
+ parallel_config=None,
137
+ scheduler_config=scheduler_config,
138
+ device_config=None,
139
+ load_config=None,
140
+ lora_config=None)
141
model_runner.set_block_size(16)
142
143
prompt_lens = []
@@ -205,14 +212,17 @@ def test_empty_seq_group():
212
"facebook/opt-125m",
213
tokenizer_mode="auto",
214
trust_remote_code=False,
208
- download_dir=None,
209
- load_format="dummy",
215
seed=0,
216
dtype="float16",
217
revision=None,
218
enforce_eager=False,
219
)
215
- model_runner = ModelRunner(model_config, None, None, None, None)
220
+ model_runner = ModelRunner(model_config=model_config,
221
+ parallel_config=None,
222
+ scheduler_config=None,
223
+ device_config=None,
224
+ load_config=None,
225
+ lora_config=None)
226
model_runner.set_block_size(16)
227
seq_group_metadata_list = []
228
input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
@@ -251,8 +261,6 @@ def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
261
"facebook/opt-125m",
262
tokenizer_mode="auto",
263
trust_remote_code=False,
254
- download_dir=None,
255
- load_format="dummy",
264
seed=0,
265
dtype="float16",
266
revision=None,
@@ -262,11 +270,12 @@ def test_hybrid_batches(batch_size, enforce_eager, monkeypatch):
270
100000,
271
100000,
272
enable_chunked_prefill=True)
265
- model_runner = ModelRunner(model_config,
266
- None,
267
- scheduler_config,
268
- None,
269
- None,
273
+ model_runner = ModelRunner(model_config=model_config,
274
+ parallel_config=None,
275
+ scheduler_config=scheduler_config,
276
+ device_config=None,
277
+ load_config=None,
278
+ lora_config=None,
279
is_driver_worker=True)
280
model_runner.set_block_size(16)
281
tests/worker/test_swap.py
+1
@@ -23,6 +23,7 @@ def test_swap() -> None:
23
scheduler_config=engine_config.scheduler_config,
24
device_config=engine_config.device_config,
25
cache_config=engine_config.cache_config,
26
+ load_config=engine_config.load_config,
27
local_rank=0,
28
rank=0,
29
distributed_init_method=distributed_init_method,
vllm/config.py
+66
-135
@@ -1,9 +1,7 @@
1
import enum
2
-import io
2
import json
3
import os
5
-import typing
6
-from dataclasses import dataclass, fields
4
+from dataclasses import dataclass, field, fields
5
from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
6
7
import torch
@@ -18,10 +16,14 @@ from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
16
if TYPE_CHECKING:
17
from ray.util.placement_group import PlacementGroup
18
21
- from vllm.model_executor.tensorizer_loader import TensorizerArgs
19
+ from vllm.model_executor.model_loader.loader import BaseModelLoader
20
21
logger = init_logger(__name__)
22
23
+# If true, will load models from ModelScope instead of Hugging Face Hub.
24
+VLLM_USE_MODELSCOPE = os.environ.get("VLLM_USE_MODELSCOPE",
25
+ "False").lower() == "true"
26
+
27
_GB = 1 << 30
28
29
@@ -35,18 +37,6 @@ class ModelConfig:
37
available, and "slow" will always use the slow tokenizer.
38
trust_remote_code: Trust remote code (e.g., from HuggingFace) when
39
downloading the model and tokenizer.
38
- download_dir: Directory to download and load the weights, default to the
39
- default cache directory of huggingface.
40
- load_format: The format of the model weights to load:
41
- "auto" will try to load the weights in the safetensors format and
42
- fall back to the pytorch bin format if safetensors format is
43
- not available.
44
- "pt" will load the weights in the pytorch bin format.
45
- "safetensors" will load the weights in the safetensors format.
46
- "npcache" will load the weights in pytorch format and store
47
- a numpy cache to speed up the loading.
48
- "dummy" will initialize the weights with random values, which is
49
- mainly for profiling.
40
dtype: Data type for model weights and activations. The "auto" option
41
will use FP16 precision for FP32 and FP16 models, and BF16 precision
42
for BF16 models.
@@ -83,8 +73,6 @@ class ModelConfig:
73
tokenizer: str,
74
tokenizer_mode: str,
75
trust_remote_code: bool,
86
- download_dir: Optional[str],
87
- load_format: str,
76
dtype: Union[str, torch.dtype],
77
seed: int,
78
revision: Optional[str] = None,
@@ -101,8 +89,6 @@ class ModelConfig:
89
self.tokenizer = tokenizer
90
self.tokenizer_mode = tokenizer_mode
91
self.trust_remote_code = trust_remote_code
104
- self.download_dir = download_dir
105
- self.load_format = load_format
92
self.seed = seed
93
self.revision = revision
94
self.code_revision = code_revision
@@ -113,64 +99,16 @@ class ModelConfig:
99
self.max_context_len_to_capture = max_context_len_to_capture
100
self.max_logprobs = max_logprobs
101
116
- if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
117
- # download model from ModelScope hub,
118
- # lazy import so that modelscope is not required for normal use.
119
- # pylint: disable=C.
120
- from modelscope.hub.snapshot_download import snapshot_download
121
-
122
- if not os.path.exists(model):
123
- model_path = snapshot_download(model_id=model,
124
- cache_dir=download_dir,
125
- revision=revision)
126
- else:
127
- model_path = model
128
- self.model = model_path
129
- self.download_dir = model_path
130
- self.tokenizer = model_path
131
-
102
self.hf_config = get_config(self.model, trust_remote_code, revision,
103
code_revision)
104
self.hf_text_config = get_hf_text_config(self.hf_config)
105
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
106
self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
107
max_model_len)
138
- self._verify_load_format()
108
self._verify_tokenizer_mode()
109
self._verify_quantization()
110
self._verify_cuda_graph()
111
143
- def _verify_load_format(self) -> None:
144
- load_format = self.load_format.lower()
145
- supported_load_format = [
146
- "auto", "pt", "safetensors", "npcache", "dummy", "tensorizer"
147
- ]
148
- rocm_not_supported_load_format: List[str] = []
149
- if load_format not in supported_load_format:
150
- raise ValueError(
151
- f"Unknown load format: {self.load_format}. Must be one of "
152
- "'auto', 'pt', 'safetensors', 'npcache', 'tensorizer', or "
153
- "'dummy'.")
154
- if is_hip() and load_format in rocm_not_supported_load_format:
155
- rocm_supported_load_format = [
156
- f for f in supported_load_format
157
- if (f not in rocm_not_supported_load_format)
158
- ]
159
- raise ValueError(
160
- f"load format '{load_format}' is not supported in ROCm. "
161
- f"Supported load format are "
162
- f"{rocm_supported_load_format}")
163
-
164
- # TODO: Remove this check once HF updates the pt weights of Mixtral.
165
- architectures = getattr(self.hf_config, "architectures", [])
166
- # architectures can be None instead of []
167
- if architectures and "MixtralForCausalLM" in architectures \
168
- and load_format == "pt":
169
- raise ValueError(
170
- "Currently, the 'pt' format is not supported for Mixtral. "
171
- "Please use the 'safetensors' format instead. ")
172
- self.load_format = load_format
173
-
112
def _verify_tokenizer_mode(self) -> None:
113
tokenizer_mode = self.tokenizer_mode.lower()
114
if tokenizer_mode not in ["auto", "slow"]:
@@ -471,6 +409,65 @@ class TokenizerPoolConfig:
409
return tokenizer_pool_config
410
411
412
+class LoadFormat(str, enum.Enum):
413
+ AUTO = "auto"
414
+ PT = "pt"
415
+ SAFETENSORS = "safetensors"
416
+ NPCACHE = "npcache"
417
+ DUMMY = "dummy"
418
+ TENSORIZER = "tensorizer"
419
+
420
+
421
+@dataclass
422
+class LoadConfig:
423
+ """
424
+ download_dir: Directory to download and load the weights, default to the
425
+ default cache directory of huggingface.
426
+ load_format: The format of the model weights to load:
427
+ "auto" will try to load the weights in the safetensors format and
428
+ fall back to the pytorch bin format if safetensors format is
429
+ not available.
430
+ "pt" will load the weights in the pytorch bin format.
431
+ "safetensors" will load the weights in the safetensors format.
432
+ "npcache" will load the weights in pytorch format and store
433
+ a numpy cache to speed up the loading.
434
+ "dummy" will initialize the weights with random values, which is
435
+ mainly for profiling.
436
+ "tensorizer" will use CoreWeave's tensorizer library for
437
+ fast weight loading.
438
+ """
439
+
440
+ load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO
441
+ download_dir: Optional[str] = None
442
+ model_loader_extra_config: Optional[Union[str, dict]] = field(
443
+ default_factory=dict)
444
+
445
+ def __post_init__(self):
446
+ model_loader_extra_config = self.model_loader_extra_config or {}
447
+ if isinstance(model_loader_extra_config, str):
448
+ self.model_loader_extra_config = json.loads(
449
+ model_loader_extra_config)
450
+ self._verify_load_format()
451
+
452
+ def _verify_load_format(self) -> None:
453
+ if not isinstance(self.load_format, str):
454
+ return
455
+
456
+ load_format = self.load_format.lower()
457
+ self.load_format = LoadFormat(load_format)
458
+
459
+ rocm_not_supported_load_format: List[str] = []
460
+ if is_hip() and load_format in rocm_not_supported_load_format:
461
+ rocm_supported_load_format = [
462
+ f for f in LoadFormat.__members__
463
+ if (f not in rocm_not_supported_load_format)
464
+ ]
465
+ raise ValueError(
466
+ f"load format '{load_format}' is not supported in ROCm. "
467
+ f"Supported load formats are "
468
+ f"{rocm_supported_load_format}")
469
+
470
+
471
class ParallelConfig:
472
"""Configuration for the distributed execution.
473
@@ -699,8 +696,6 @@ class SpeculativeConfig:
696
tokenizer=target_model_config.tokenizer,
697
tokenizer_mode=target_model_config.tokenizer_mode,
698
trust_remote_code=target_model_config.trust_remote_code,
702
- download_dir=target_model_config.download_dir,
703
- load_format=target_model_config.load_format,
699
dtype=target_model_config.dtype,
700
seed=target_model_config.seed,
701
revision=draft_revision,
@@ -887,65 +882,6 @@ class VisionLanguageConfig:
882
f"{[x.name for x in cls.ImageInputType]}.") from e
883
884
890
-@dataclass
891
-class TensorizerConfig:
892
- tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
893
- str, bytes, os.PathLike, int]
894
- vllm_tensorized: bool
895
- verify_hash: Optional[bool] = False
896
- num_readers: Optional[int] = 1
897
- encryption_keyfile: Optional[str] = None
898
- s3_access_key_id: Optional[str] = None
899
- s3_secret_access_key: Optional[str] = None
900
- s3_endpoint: Optional[str] = None
901
- model_class: Optional[torch.nn.Module] = None
902
- hf_config: Optional[PretrainedConfig] = None
903
- dtype: Union[str, torch.dtype] = None
904
-
905
- def _construct_tensorizer_args(self) -> "TensorizerArgs":
906
- from vllm.model_executor.tensorizer_loader import TensorizerArgs
907
- tensorizer_args = {
908
- "tensorizer_uri": self.tensorizer_uri,
909
- "vllm_tensorized": self.vllm_tensorized,
910
- "verify_hash": self.verify_hash,
911
- "num_readers": self.num_readers,
912
- "encryption_keyfile": self.encryption_keyfile,
913
- "s3_access_key_id": self.s3_access_key_id,
914
- "s3_secret_access_key": self.s3_secret_access_key,
915
- "s3_endpoint": self.s3_endpoint,
916
- }
917
- return TensorizerArgs(**tensorizer_args)
918
-
919
- def verify_with_parallel_config(
920
- self,
921
- parallel_config: "ParallelConfig",
922
- ) -> None:
923
- if (parallel_config.tensor_parallel_size > 1
924
- and self.tensorizer_uri is not None):
925
- raise ValueError(
926
- "Loading to multiple GPUs is not currently supported with "
927
- "vLLM-serialized models. Please set tensor_parallel_size=1."
928
- " or use a non-vLLM-serialized model, such as a "
929
- "serialized Hugging Face `PretrainedModel`.")
930
-
931
- def verify_with_model_config(self, model_config) -> None:
932
- if (model_config.quantization is not None
933
- and self.tensorizer_uri is not None):
934
- from vllm.model_executor.tensorizer_loader import (
935
- tensorizer_warning)
936
- tensorizer_warning(
937
- "Loading a model using Tensorizer with quantization on vLLM"
938
- " is unstable and may lead to errors.")
939
-
940
- if (model_config.load_format != "tensorizer"
941
- and self.tensorizer_uri is not None):
942
- raise ValueError(
943
- "A tensorizer uri was passed for tensorizer loading, but the "
944
- f"load format was set to {model_config.load_format}. "
945
- "Please set the load format to 'tensorizer' to use "
946
- f"tensorizer args.")
947
-
948
-
885
_STR_DTYPE_TO_TORCH_DTYPE = {
886
"half": torch.float16,
887
"float16": torch.float16,
@@ -1105,11 +1041,11 @@ class EngineConfig:
1041
parallel_config: ParallelConfig
1042
scheduler_config: SchedulerConfig
1043
device_config: DeviceConfig
1044
+ load_config: LoadConfig
1045
lora_config: Optional[LoRAConfig]
1046
vision_language_config: Optional[VisionLanguageConfig]
1047
speculative_config: Optional[SpeculativeConfig]
1048
decoding_config: Optional[DecodingConfig]
1112
- tensorizer_config: Optional[TensorizerConfig]
1049
1050
def __post_init__(self):
1051
"""Verify configs are valid & consistent with each other.
@@ -1117,11 +1053,6 @@ class EngineConfig:
1053
self.model_config.verify_with_parallel_config(self.parallel_config)
1054
self.cache_config.verify_with_parallel_config(self.parallel_config)
1055
1120
- if self.tensorizer_config:
1121
- self.tensorizer_config.verify_with_parallel_config(
1122
- self.parallel_config)
1123
- self.tensorizer_config.verify_with_model_config(self.model_config)
1124
-
1056
if self.lora_config:
1057
self.lora_config.verify_with_model_config(self.model_config)
1058
self.lora_config.verify_with_scheduler_config(
vllm/engine/arg_utils.py
+25
-34
@@ -1,15 +1,12 @@
1
import argparse
2
import dataclasses
3
-import io
4
-import os
3
from dataclasses import dataclass
6
-from typing import BinaryIO, Optional, Union
4
+from typing import Optional
5
6
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9
- EngineConfig, LoRAConfig, ModelConfig, ParallelConfig,
10
- SchedulerConfig, SpeculativeConfig, TensorizerConfig,
7
+ EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
8
+ ParallelConfig, SchedulerConfig, SpeculativeConfig,
9
TokenizerPoolConfig, VisionLanguageConfig)
12
-from vllm.model_executor.tensorizer_loader import TensorizerArgs
10
from vllm.utils import str_to_int_tuple
11
12
@@ -60,17 +57,7 @@ class EngineArgs:
57
ray_workers_use_nsight: bool = False
58
num_gpu_blocks_override: Optional[int] = None
59
num_lookahead_slots: int = 0
63
-
64
- # Tensorizer configuration parameters
65
- tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, BinaryIO, str,
66
- bytes, os.PathLike, int] = None
67
- vllm_tensorized: bool = False
68
- verify_hash: Optional[bool] = False
69
- num_readers: Optional[int] = 1
70
- encryption_keyfile: Optional[str] = None
71
- s3_access_key_id: Optional[str] = None
72
- s3_secret_access_key: Optional[str] = None
73
- s3_endpoint: Optional[str] = None
60
+ model_loader_extra_config: Optional[dict] = None
61
62
# Related to Vision-language models such as llava
63
image_input_type: Optional[str] = None
@@ -429,7 +416,16 @@ class EngineArgs:
416
default=None,
417
help='The number of speculative tokens to sample from '
418
'the draft model in speculative decoding')
432
- parser = TensorizerArgs.add_cli_args(parser)
419
+
420
+ parser.add_argument('--model-loader-extra-config',
421
+ type=str,
422
+ default=EngineArgs.model_loader_extra_config,
423
+ help='Extra config for model loader. '
424
+ 'This will be passed to the model loader '
425
+ 'corresponding to the chosen load_format. '
426
+ 'This should be a JSON string that will be '
427
+ 'parsed into a dictionary.')
428
+
429
return parser
430
431
@classmethod
@@ -444,11 +440,11 @@ class EngineArgs:
440
device_config = DeviceConfig(self.device)
441
model_config = ModelConfig(
442
self.model, self.tokenizer, self.tokenizer_mode,
447
- self.trust_remote_code, self.download_dir, self.load_format,
448
- self.dtype, self.seed, self.revision, self.code_revision,
449
- self.tokenizer_revision, self.max_model_len, self.quantization,
450
- self.quantization_param_path, self.enforce_eager,
451
- self.max_context_len_to_capture, self.max_logprobs)
443
+ self.trust_remote_code, self.dtype, self.seed, self.revision,
444
+ self.code_revision, self.tokenizer_revision, self.max_model_len,
445
+ self.quantization, self.quantization_param_path,
446
+ self.enforce_eager, self.max_context_len_to_capture,
447
+ self.max_logprobs)
448
cache_config = CacheConfig(self.block_size,
449
self.gpu_memory_utilization,
450
self.swap_space, self.kv_cache_dtype,
@@ -492,15 +488,10 @@ class EngineArgs:
488
max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
489
and self.max_cpu_loras > 0 else None) if self.enable_lora else None
490
495
- tensorizer_config = TensorizerConfig(
496
- tensorizer_uri=self.tensorizer_uri,
497
- vllm_tensorized=self.vllm_tensorized,
498
- verify_hash=self.verify_hash,
499
- num_readers=self.num_readers,
500
- encryption_keyfile=self.encryption_keyfile,
501
- s3_access_key_id=self.s3_access_key_id,
502
- s3_secret_access_key=self.s3_secret_access_key,
503
- s3_endpoint=self.s3_endpoint,
491
+ load_config = LoadConfig(
492
+ load_format=self.load_format,
493
+ download_dir=self.download_dir,
494
+ model_loader_extra_config=self.model_loader_extra_config,
495
)
496
497
if self.image_input_type:
@@ -530,8 +521,8 @@ class EngineArgs:
521
lora_config=lora_config,
522
vision_language_config=vision_language_config,
523
speculative_config=speculative_config,
533
- decoding_config=decoding_config,
534
- tensorizer_config=tensorizer_config)
524
+ load_config=load_config,
525
+ decoding_config=decoding_config)
526
527
528
@dataclass
vllm/engine/llm_engine.py
+8
-11
@@ -4,9 +4,9 @@ from typing import Iterable, List, Optional, Tuple, Type, Union
4
from transformers import PreTrainedTokenizer
5
6
import vllm
7
-from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoRAConfig,
8
- ModelConfig, ParallelConfig, SchedulerConfig,
9
- SpeculativeConfig, TensorizerConfig,
7
+from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
8
+ LoRAConfig, ModelConfig, ParallelConfig,
9
+ SchedulerConfig, SpeculativeConfig,
10
VisionLanguageConfig)
11
from vllm.core.scheduler import Scheduler, SchedulerOutputs
12
from vllm.engine.arg_utils import EngineArgs
@@ -72,11 +72,11 @@ class LLMEngine:
72
parallel_config: ParallelConfig,
73
scheduler_config: SchedulerConfig,
74
device_config: DeviceConfig,
75
+ load_config: LoadConfig,
76
lora_config: Optional[LoRAConfig],
77
vision_language_config: Optional[VisionLanguageConfig],
78
speculative_config: Optional[SpeculativeConfig],
79
decoding_config: Optional[DecodingConfig],
79
- tensorizer_config: Optional[TensorizerConfig],
80
executor_class: Type[ExecutorBase],
81
log_stats: bool,
82
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -92,8 +92,8 @@ class LLMEngine:
92
f"trust_remote_code={model_config.trust_remote_code}, "
93
f"dtype={model_config.dtype}, "
94
f"max_seq_len={model_config.max_model_len}, "
95
- f"download_dir={model_config.download_dir!r}, "
96
- f"load_format={model_config.load_format}, "
95
+ f"download_dir={load_config.download_dir!r}, "
96
+ f"load_format={load_config.load_format}, "
97
f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
98
f"disable_custom_all_reduce="
99
f"{parallel_config.disable_custom_all_reduce}, "
@@ -114,8 +114,8 @@ class LLMEngine:
114
self.scheduler_config = scheduler_config
115
self.device_config = device_config
116
self.speculative_config = speculative_config
117
+ self.load_config = load_config
118
self.decoding_config = decoding_config or DecodingConfig()
118
- self.tensorizer_config = tensorizer_config
119
self.log_stats = log_stats
120
121
self._init_tokenizer()
@@ -131,7 +131,7 @@ class LLMEngine:
131
lora_config=lora_config,
132
vision_language_config=vision_language_config,
133
speculative_config=speculative_config,
134
- tensorizer_config=tensorizer_config,
134
+ load_config=load_config,
135
)
136
137
self._initialize_kv_caches()
@@ -271,9 +271,6 @@ class LLMEngine:
271
def _verify_args(self) -> None:
272
self.model_config.verify_with_parallel_config(self.parallel_config)
273
self.cache_config.verify_with_parallel_config(self.parallel_config)
274
- if self.tensorizer_config:
275
- self.tensorizer_config.verify_with_parallel_config(
276
- self.parallel_config)
274
if self.lora_config:
275
self.lora_config.verify_with_model_config(self.model_config)
276
self.lora_config.verify_with_scheduler_config(
vllm/executor/cpu_executor.py
+1
@@ -40,6 +40,7 @@ class CPUExecutor(ExecutorBase):
40
scheduler_config=self.scheduler_config,
41
device_config=self.device_config,
42
cache_config=self.cache_config,
43
+ load_config=self.load_config,
44
local_rank=0,
45
rank=0,
46
distributed_init_method=distributed_init_method,
vllm/executor/executor_base.py
+5
-5
@@ -1,9 +1,9 @@
1
from abc import ABC, abstractmethod
2
from typing import Dict, List, Optional, Set, Tuple
3
4
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5
- ParallelConfig, SchedulerConfig, SpeculativeConfig,
6
- TensorizerConfig, VisionLanguageConfig)
4
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5
+ ModelConfig, ParallelConfig, SchedulerConfig,
6
+ SpeculativeConfig, VisionLanguageConfig)
7
from vllm.lora.request import LoRARequest
8
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
@@ -23,20 +23,20 @@ class ExecutorBase(ABC):
23
parallel_config: ParallelConfig,
24
scheduler_config: SchedulerConfig,
25
device_config: DeviceConfig,
26
+ load_config: LoadConfig,
27
lora_config: Optional[LoRAConfig],
28
vision_language_config: Optional[VisionLanguageConfig],
29
speculative_config: Optional[SpeculativeConfig],
29
- tensorizer_config: Optional[TensorizerConfig],
30
) -> None:
31
self.model_config = model_config
32
self.cache_config = cache_config
33
self.lora_config = lora_config
34
+ self.load_config = load_config
35
self.parallel_config = parallel_config
36
self.scheduler_config = scheduler_config
37
self.device_config = device_config
38
self.vision_language_config = vision_language_config
39
self.speculative_config = speculative_config
39
- self.tensorizer_config = tensorizer_config
40
41
self._init_executor()
42
vllm/executor/gpu_executor.py
+1
-1
@@ -35,12 +35,12 @@ class GPUExecutor(ExecutorBase):
35
scheduler_config=self.scheduler_config,
36
device_config=self.device_config,
37
cache_config=self.cache_config,
38
+ load_config=self.load_config,
39
local_rank=0,
40
rank=0,
41
distributed_init_method=distributed_init_method,
42
lora_config=self.lora_config,
43
vision_language_config=self.vision_language_config,
43
- tensorizer_config=self.tensorizer_config,
44
is_driver_worker=True,
45
)
46
self.driver_worker.init_device()
vllm/executor/ray_gpu_executor.py
+3
-2
@@ -147,6 +147,7 @@ class RayGPUExecutor(ExecutorBase):
147
model_config = copy.deepcopy(self.model_config)
148
parallel_config = copy.deepcopy(self.parallel_config)
149
scheduler_config = copy.deepcopy(self.scheduler_config)
150
+ load_config = copy.deepcopy(self.load_config)
151
device_config = copy.deepcopy(self.device_config)
152
lora_config = copy.deepcopy(self.lora_config)
153
cache_config = copy.deepcopy(self.cache_config)
@@ -165,12 +166,12 @@ class RayGPUExecutor(ExecutorBase):
166
scheduler_config=scheduler_config,
167
device_config=device_config,
168
cache_config=cache_config,
169
+ load_config=load_config,
170
local_rank=local_rank,
171
rank=rank,
172
distributed_init_method=distributed_init_method,
173
lora_config=lora_config,
174
vision_language_config=vision_language_config,
173
- tensorizer_config=self.tensorizer_config,
175
))
176
177
# Initialize the driver worker with the Worker class.
@@ -187,7 +188,7 @@ class RayGPUExecutor(ExecutorBase):
188
distributed_init_method=distributed_init_method,
189
lora_config=self.lora_config,
190
vision_language_config=self.vision_language_config,
190
- tensorizer_config=self.tensorizer_config,
191
+ load_config=self.load_config,
192
is_driver_worker=True,
193
)
194
vllm/model_executor/model_loader.py
deleted
-128
@@ -1,128 +0,0 @@
1
-"""Utilities for selecting and loading models."""
2
-import contextlib
3
-from typing import Tuple, Type
4
-
5
-import torch
6
-from torch import nn
7
-
8
-from vllm.config import DeviceConfig, ModelConfig
9
-from vllm.model_executor.models import ModelRegistry
10
-from vllm.model_executor.models.llava import LlavaForConditionalGeneration
11
-from vllm.model_executor.tensorizer_loader import (
12
- ParameterizedLoadFormat, is_vllm_serialized_tensorizer,
13
- load_with_tensorizer)
14
-from vllm.model_executor.weight_utils import (get_quant_config,
15
- initialize_dummy_weights)
16
-
17
-_VISION_MODEL_CLASSES = [
18
- LlavaForConditionalGeneration,
19
-]
20
-
21
-
22
-@contextlib.contextmanager
23
-def _set_default_torch_dtype(dtype: torch.dtype):
24
- """Sets the default torch dtype to the given dtype."""
25
- old_dtype = torch.get_default_dtype()
26
- torch.set_default_dtype(dtype)
27
- yield
28
- torch.set_default_dtype(old_dtype)
29
-
30
-
31
-def _get_model_architecture(
32
- model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
33
- architectures = getattr(model_config.hf_config, "architectures", [])
34
- # Special handling for quantized Mixtral.
35
- # FIXME(woosuk): This is a temporary hack.
36
- if (model_config.quantization is not None
37
- and "MixtralForCausalLM" in architectures):
38
- architectures = ["QuantMixtralForCausalLM"]
39
-
40
- for arch in architectures:
41
- model_cls = ModelRegistry.load_model_cls(arch)
42
- if model_cls is not None:
43
- return (model_cls, arch)
44
- raise ValueError(
45
- f"Model architectures {architectures} are not supported for now. "
46
- f"Supported architectures: {ModelRegistry.get_supported_archs()}")
47
-
48
-
49
-def get_architecture_class_name(model_config: ModelConfig) -> str:
50
- return _get_model_architecture(model_config)[1]
51
-
52
-
53
-def get_model(model_config: ModelConfig, device_config: DeviceConfig,
54
- **kwargs) -> nn.Module:
55
- lora_config = kwargs.get("lora_config", None)
56
- vision_language_config = kwargs.get("vision_language_config", None)
57
- tensorizer_config = kwargs.get("tensorizer_config", None)
58
- model_class = _get_model_architecture(model_config)[0]
59
-
60
- # Get the (maybe quantized) linear method.
61
- linear_method = None
62
- if model_config.quantization is not None:
63
- quant_config = get_quant_config(model_config)
64
- capability = torch.cuda.get_device_capability()
65
- capability = capability[0] * 10 + capability[1]
66
- if capability < quant_config.get_min_capability():
67
- raise ValueError(
68
- f"The quantization method {model_config.quantization} is not "
69
- "supported for the current GPU. "
70
- f"Minimum capability: {quant_config.get_min_capability()}. "
71
- f"Current capability: {capability}.")
72
- supported_dtypes = quant_config.get_supported_act_dtypes()
73
- if model_config.dtype not in supported_dtypes:
74
- raise ValueError(
75
- f"{model_config.dtype} is not supported for quantization "
76
- f"method {model_config.quantization}. Supported dtypes: "
77
- f"{supported_dtypes}")
78
-
79
- linear_method = quant_config.get_linear_method()
80
-
81
- with _set_default_torch_dtype(model_config.dtype):
82
- # Create a model instance.
83
- # The weights will be initialized as empty tensors.
84
- extra_kwargs = {}
85
- if hasattr(model_class, "supported_lora_modules"):
86
- extra_kwargs["lora_config"] = lora_config
87
- elif lora_config:
88
- raise ValueError(
89
- f"Model {model_class.__name__} does not support LoRA, "
90
- "but LoRA is enabled. Support for this model may "
91
- "be added in the future. If this is important to you, "
92
- "please open an issue on github.")
93
- elif model_class in _VISION_MODEL_CLASSES:
94
- extra_kwargs["vision_language_config"] = vision_language_config
95
-
96
- with torch.device(device_config.device):
97
- if (model_config.load_format == "tensorizer"
98
- and is_vllm_serialized_tensorizer(tensorizer_config)):
99
- extra_kwargs["linear_method"] = linear_method
100
- tensorizer_config.model_class = model_class
101
- tensorizer_config.hf_config = model_config.hf_config
102
- tensorizer_config.dtype = model_config.dtype
103
- model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
104
- return model.eval()
105
- model = model_class(config=model_config.hf_config,
106
- linear_method=linear_method,
107
- **extra_kwargs)
108
- if model_config.load_format == "dummy":
109
- # NOTE(woosuk): For accurate performance evaluation, we assign
110
- # random values to the weights.
111
- initialize_dummy_weights(model)
112
- else:
113
- # Load the weights from the cached or downloaded files.
114
- if model_config.load_format == "tensorizer":
115
- # Provide a dynamic load format for `model.load_weights`
116
- # to retain tensorizer args from CLI.
117
- model_config.load_format = ParameterizedLoadFormat(
118
- model_config.load_format)
119
- model_config.load_format.params = (
120
- tensorizer_config._construct_tensorizer_args())
121
-
122
- model.load_weights(
123
- model_config.model,
124
- model_config.download_dir,
125
- model_config.load_format,
126
- model_config.revision,
127
- )
128
- return model.eval()
vllm/model_executor/model_loader/__init__.py
new
+30
@@ -0,0 +1,30 @@
1
+from typing import Optional
2
+
3
+from torch import nn
4
+
5
+from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
6
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
7
+from vllm.model_executor.model_loader.loader import (BaseModelLoader,
8
+ get_model_loader)
9
+from vllm.model_executor.model_loader.utils import (
10
+ get_architecture_class_name, get_model_architecture)
11
+
12
+
13
+def get_model(
14
+ *, model_config: ModelConfig, load_config: LoadConfig,
15
+ device_config: DeviceConfig, parallel_config: ParallelConfig,
16
+ scheduler_config: SchedulerConfig, lora_config: Optional[LoRAConfig],
17
+ vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
18
+ loader = get_model_loader(load_config)
19
+ return loader.load_model(model_config=model_config,
20
+ device_config=device_config,
21
+ lora_config=lora_config,
22
+ vision_language_config=vision_language_config,
23
+ parallel_config=parallel_config,
24
+ scheduler_config=scheduler_config)
25
+
26
+
27
+__all__ = [
28
+ "get_model", "get_model_loader", "BaseModelLoader",
29
+ "get_architecture_class_name", "get_model_architecture"
30
+]
vllm/model_executor/model_loader/loader.py
new
+354
@@ -0,0 +1,354 @@
1
+# ruff: noqa: SIM117
2
+import copy
3
+import glob
4
+import os
5
+from abc import ABC, abstractmethod
6
+from typing import (TYPE_CHECKING, Any, Dict, Generator, List, Optional, Tuple,
7
+ Type)
8
+
9
+import torch
10
+from torch import nn
11
+
12
+from vllm.config import (VLLM_USE_MODELSCOPE, DeviceConfig, LoadConfig,
13
+ LoadFormat, LoRAConfig, ModelConfig, ParallelConfig,
14
+ SchedulerConfig, VisionLanguageConfig)
15
+from vllm.logger import init_logger
16
+from vllm.model_executor.model_loader.tensorizer import (
17
+ TensorizerConfig, is_vllm_serialized_tensorizer, load_with_tensorizer,
18
+ tensorizer_weights_iterator)
19
+from vllm.model_executor.model_loader.utils import (get_model_architecture,
20
+ set_default_torch_dtype)
21
+from vllm.model_executor.model_loader.weight_utils import (
22
+ download_weights_from_hf, filter_files_not_needed_for_inference,
23
+ get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
24
+ pt_weights_iterator, safetensors_weights_iterator)
25
+from vllm.model_executor.models.llava import LlavaForConditionalGeneration
26
+
27
+if TYPE_CHECKING:
28
+ from vllm.model_executor.layers.linear import LinearMethodBase
29
+
30
+_VISION_MODEL_CLASSES = [
31
+ LlavaForConditionalGeneration,
32
+]
33
+
34
+logger = init_logger(__name__)
35
+
36
+
37
+def _get_linear_method(
38
+ model_config: ModelConfig,
39
+ load_config: LoadConfig) -> Optional["LinearMethodBase"]:
40
+ """Get the (maybe quantized) linear method."""
41
+ linear_method = None
42
+ if model_config.quantization is not None:
43
+ quant_config = get_quant_config(model_config, load_config)
44
+ capability = torch.cuda.get_device_capability()
45
+ capability = capability[0] * 10 + capability[1]
46
+ if capability < quant_config.get_min_capability():
47
+ raise ValueError(
48
+ f"The quantization method {model_config.quantization} is not "
49
+ "supported for the current GPU. "
50
+ f"Minimum capability: {quant_config.get_min_capability()}. "
51
+ f"Current capability: {capability}.")
52
+ supported_dtypes = quant_config.get_supported_act_dtypes()
53
+ if model_config.dtype not in supported_dtypes:
54
+ raise ValueError(
55
+ f"{model_config.dtype} is not supported for quantization "
56
+ f"method {model_config.quantization}. Supported dtypes: "
57
+ f"{supported_dtypes}")
58
+
59
+ linear_method = quant_config.get_linear_method()
60
+ return linear_method
61
+
62
+
63
+def _get_model_initialization_kwargs(
64
+ model_class: Type[nn.Module], lora_config: Optional[LoRAConfig],
65
+ vision_language_config: Optional[VisionLanguageConfig]
66
+) -> Dict[str, Any]:
67
+ """Get extra kwargs for model initialization."""
68
+ extra_kwargs = {}
69
+ if hasattr(model_class, "supported_lora_modules"):
70
+ extra_kwargs["lora_config"] = lora_config
71
+ elif lora_config:
72
+ raise ValueError(
73
+ f"Model {model_class.__name__} does not support LoRA, "
74
+ "but LoRA is enabled. Support for this model may "
75
+ "be added in the future. If this is important to you, "
76
+ "please open an issue on github.")
77
+ elif model_class in _VISION_MODEL_CLASSES:
78
+ extra_kwargs["vision_language_config"] = vision_language_config
79
+ return extra_kwargs
80
+
81
+
82
+def _initialize_model(
83
+ model_config: ModelConfig, load_config: LoadConfig,
84
+ lora_config: Optional[LoRAConfig],
85
+ vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
86
+ """Initialize a model with the given configurations."""
87
+ model_class = get_model_architecture(model_config)[0]
88
+ linear_method = _get_linear_method(model_config, load_config)
89
+
90
+ return model_class(config=model_config.hf_config,
91
+ linear_method=linear_method,
92
+ **_get_model_initialization_kwargs(
93
+ model_class, lora_config, vision_language_config))
94
+
95
+
96
+class BaseModelLoader(ABC):
97
+ """Base class for model loaders."""
98
+
99
+ def __init__(self, load_config: LoadConfig):
100
+ self.load_config = load_config
101
+
102
+ @abstractmethod
103
+ def load_model(self, *, model_config: ModelConfig,
104
+ device_config: DeviceConfig,
105
+ lora_config: Optional[LoRAConfig],
106
+ vision_language_config: Optional[VisionLanguageConfig],
107
+ parallel_config: ParallelConfig,
108
+ scheduler_config: SchedulerConfig) -> nn.Module:
109
+ """Load a model with the given configurations."""
110
+ ...
111
+
112
+
113
+class DefaultModelLoader(BaseModelLoader):
114
+ """Model loader that can load different file types from disk."""
115
+
116
+ def __init__(self, load_config: LoadConfig):
117
+ super().__init__(load_config)
118
+ if load_config.model_loader_extra_config:
119
+ raise ValueError(f"Model loader extra config is not supported for "
120
+ f"load format {load_config.load_format}")
121
+
122
+ def _maybe_download_from_modelscope(
123
+ self, model: str, revision: Optional[str]) -> Optional[str]:
124
+ """Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
125
+
126
+ Returns the path to the downloaded model, or None if the model is not
127
+ downloaded from ModelScope."""
128
+ if VLLM_USE_MODELSCOPE:
129
+ # download model from ModelScope hub,
130
+ # lazy import so that modelscope is not required for normal use.
131
+ # pylint: disable=C.
132
+ from modelscope.hub.snapshot_download import snapshot_download
133
+
134
+ if not os.path.exists(model):
135
+ model_path = snapshot_download(
136
+ model_id=model,
137
+ cache_dir=self.load_config.download_dir,
138
+ revision=revision)
139
+ else:
140
+ model_path = model
141
+ return model_path
142
+ return None
143
+
144
+ def _prepare_weights(self, model_name_or_path: str,
145
+ revision: Optional[str],
146
+ fall_back_to_pt: bool) -> Tuple[str, List[str], bool]:
147
+ """Prepare weights for the model.
148
+
149
+ If the model is not local, it will be downloaded."""
150
+ model_name_or_path = self._maybe_download_from_modelscope(
151
+ model_name_or_path, revision) or model_name_or_path
152
+
153
+ is_local = os.path.isdir(model_name_or_path)
154
+ load_format = self.load_config.load_format
155
+ use_safetensors = False
156
+ # Some quantized models use .pt files for storing the weights.
157
+ if load_format == LoadFormat.AUTO:
158
+ allow_patterns = ["*.safetensors", "*.bin"]
159
+ elif load_format == LoadFormat.SAFETENSORS:
160
+ use_safetensors = True
161
+ allow_patterns = ["*.safetensors"]
162
+ elif load_format == LoadFormat.PT:
163
+ allow_patterns = ["*.pt"]
164
+ elif load_format == LoadFormat.NPCACHE:
165
+ allow_patterns = ["*.bin"]
166
+ else:
167
+ raise ValueError(f"Unknown load_format: {load_format}")
168
+
169
+ if fall_back_to_pt:
170
+ allow_patterns += ["*.pt"]
171
+
172
+ if not is_local:
173
+ hf_folder = download_weights_from_hf(model_name_or_path,
174
+ self.load_config.download_dir,
175
+ allow_patterns)
176
+ else:
177
+ hf_folder = model_name_or_path
178
+
179
+ hf_weights_files: List[str] = []
180
+ for pattern in allow_patterns:
181
+ hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
182
+ if len(hf_weights_files) > 0:
183
+ if pattern == "*.safetensors":
184
+ use_safetensors = True
185
+ break
186
+
187
+ if not use_safetensors:
188
+ hf_weights_files = filter_files_not_needed_for_inference(
189
+ hf_weights_files)
190
+
191
+ if len(hf_weights_files) == 0:
192
+ raise RuntimeError(
193
+ f"Cannot find any model weights with `{model_name_or_path}`")
194
+
195
+ return hf_folder, hf_weights_files, use_safetensors
196
+
197
+ def _get_weights_iterator(
198
+ self, model_name_or_path: str, revision: Optional[str],
199
+ fall_back_to_pt: bool
200
+ ) -> Generator[Tuple[str, torch.Tensor], None, None]:
201
+ """Get an iterator for the model weights based on the load format."""
202
+ hf_folder, hf_weights_files, use_safetensors = self._prepare_weights(
203
+ model_name_or_path, revision, fall_back_to_pt)
204
+ if self.load_config.load_format == LoadFormat.NPCACHE:
205
+ # Currently np_cache only support *.bin checkpoints
206
+ assert use_safetensors is False
207
+ return np_cache_weights_iterator(model_name_or_path,
208
+ self.load_config.download_dir,
209
+ hf_folder, hf_weights_files)
210
+ if use_safetensors:
211
+ return safetensors_weights_iterator(hf_weights_files)
212
+ return pt_weights_iterator(hf_weights_files)
213
+
214
+ def load_model(self, *, model_config: ModelConfig,
215
+ device_config: DeviceConfig,
216
+ lora_config: Optional[LoRAConfig],
217
+ vision_language_config: Optional[VisionLanguageConfig],
218
+ parallel_config: ParallelConfig,
219
+ scheduler_config: SchedulerConfig) -> nn.Module:
220
+ with set_default_torch_dtype(model_config.dtype):
221
+ with torch.device(device_config.device):
222
+ model = _initialize_model(model_config, self.load_config,
223
+ lora_config, vision_language_config)
224
+ model.load_weights(
225
+ self._get_weights_iterator(model_config.model,
226
+ model_config.revision,
227
+ fall_back_to_pt=getattr(
228
+ model,
229
+ "fall_back_to_pt_during_load",
230
+ True)), )
231
+ return model.eval()
232
+
233
+
234
+class DummyModelLoader(BaseModelLoader):
235
+ """Model loader that will set model weights to random values."""
236
+
237
+ def __init__(self, load_config: LoadConfig):
238
+ super().__init__(load_config)
239
+ if load_config.model_loader_extra_config:
240
+ raise ValueError(f"Model loader extra config is not supported for "
241
+ f"load format {load_config.load_format}")
242
+
243
+ def load_model(self, *, model_config: ModelConfig,
244
+ device_config: DeviceConfig,
245
+ lora_config: Optional[LoRAConfig],
246
+ vision_language_config: Optional[VisionLanguageConfig],
247
+ parallel_config: ParallelConfig,
248
+ scheduler_config: SchedulerConfig) -> nn.Module:
249
+ with set_default_torch_dtype(model_config.dtype):
250
+ with torch.device(device_config.device):
251
+ model = _initialize_model(model_config, self.load_config,
252
+ lora_config, vision_language_config)
253
+ # NOTE(woosuk): For accurate performance evaluation, we assign
254
+ # random values to the weights.
255
+ initialize_dummy_weights(model)
256
+ return model.eval()
257
+
258
+
259
+class TensorizerLoader(BaseModelLoader):
260
+ """Model loader using CoreWeave's tensorizer library."""
261
+
262
+ def __init__(self, load_config: LoadConfig):
263
+ super().__init__(load_config)
264
+ if isinstance(load_config.model_loader_extra_config, TensorizerConfig):
265
+ self.tensorizer_config = load_config.model_loader_extra_config
266
+ else:
267
+ self.tensorizer_config = TensorizerConfig(
268
+ **load_config.model_loader_extra_config)
269
+
270
+ def _verify_config(self, model_config: ModelConfig,
271
+ parallel_config: ParallelConfig):
272
+ self.tensorizer_config.verify_with_model_config(model_config)
273
+ self.tensorizer_config.verify_with_parallel_config(parallel_config)
274
+
275
+ def _get_weights_iterator(
276
+ self) -> Generator[Tuple[str, torch.Tensor], None, None]:
277
+ tensorizer_args = self.tensorizer_config._construct_tensorizer_args()
278
+ return tensorizer_weights_iterator(tensorizer_args)
279
+
280
+ def _load_model_unserialized(
281
+ self, model_config: ModelConfig, device_config: DeviceConfig,
282
+ lora_config: Optional[LoRAConfig],
283
+ vision_language_config: Optional[VisionLanguageConfig]
284
+ ) -> nn.Module:
285
+ """Load an unserialized model with tensorizer.
286
+
287
+ Unserialized here means "not serialized with tensorizer". This
288
+ should still be faster than default HuggingFace loading, but will
289
+ be slower than loading a tensorizer-serialized model.
290
+ """
291
+ with set_default_torch_dtype(model_config.dtype):
292
+ with torch.device(device_config.device):
293
+ model = _initialize_model(model_config, self.load_config,
294
+ lora_config, vision_language_config)
295
+
296
+ model.load_weights(self._get_weights_iterator())
297
+ return model.eval()
298
+
299
+ def _load_model_serialized(
300
+ self, model_config: ModelConfig, device_config: DeviceConfig,
301
+ lora_config: Optional[LoRAConfig],
302
+ vision_language_config: Optional[VisionLanguageConfig]
303
+ ) -> nn.Module:
304
+ """Load a serialized model with tensorizer.
305
+
306
+ See the examples/tensorize_vllm_model.py example "
307
+ script for serializing vLLM models."""
308
+ with set_default_torch_dtype(model_config.dtype):
309
+ with torch.device(device_config.device):
310
+ model_class = get_model_architecture(model_config)[0]
311
+ linear_method = _get_linear_method(model_config,
312
+ self.load_config)
313
+ extra_kwargs = _get_model_initialization_kwargs(
314
+ model_class, lora_config, vision_language_config)
315
+ extra_kwargs["linear_method"] = linear_method
316
+
317
+ tensorizer_config = copy.copy(self.tensorizer_config)
318
+ tensorizer_config.model_class = model_class
319
+ tensorizer_config.hf_config = model_config.hf_config
320
+ tensorizer_config.dtype = model_config.dtype
321
+
322
+ model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
323
+ return model.eval()
324
+
325
+ def load_model(self, *, model_config: ModelConfig,
326
+ device_config: DeviceConfig,
327
+ lora_config: Optional[LoRAConfig],
328
+ vision_language_config: Optional[VisionLanguageConfig],
329
+ parallel_config: ParallelConfig,
330
+ scheduler_config: SchedulerConfig) -> nn.Module:
331
+ self._verify_config(model_config, parallel_config)
332
+
333
+ if is_vllm_serialized_tensorizer(self.tensorizer_config):
334
+ return self._load_model_serialized(model_config, device_config,
335
+ lora_config,
336
+ vision_language_config)
337
+ return self._load_model_unserialized(model_config, device_config,
338
+ lora_config,
339
+ vision_language_config)
340
+
341
+
342
+def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
343
+ """Get a model loader based on the load format."""
344
+
345
+ if isinstance(load_config.load_format, type):
346
+ return load_config.load_format(load_config)
347
+
348
+ if load_config.load_format == LoadFormat.DUMMY:
349
+ return DummyModelLoader(load_config)
350
+
351
+ if load_config.load_format == LoadFormat.TENSORIZER:
352
+ return TensorizerLoader(load_config)
353
+
354
+ return DefaultModelLoader(load_config)
vllm/model_executor/model_loader/neuron.py
renamed
vllm/model_executor/model_loader/tensorizer.py
renamed
+82
-34
@@ -4,20 +4,20 @@ import io
4
import os
5
import time
6
import typing
7
-import warnings
7
from dataclasses import dataclass
9
-from typing import Optional, Union
8
+from typing import Generator, Optional, Tuple, Type, Union
9
10
import torch
11
from torch import nn
12
+from transformers import PretrainedConfig
13
14
-from vllm.config import TensorizerConfig
14
+from vllm.config import ModelConfig, ParallelConfig
15
from vllm.logger import init_logger
16
from vllm.model_executor.layers.linear import LinearMethodBase
17
from vllm.model_executor.layers.vocab_parallel_embedding import (
18
VocabParallelEmbedding)
19
20
-tensorizer_load_fail = False
20
+tensorizer_load_fail = None
21
22
try:
23
from tensorizer import (DecryptionParams, EncryptionParams,
@@ -25,51 +25,78 @@ try:
25
from tensorizer.stream_io import open_stream
26
from tensorizer.utils import (convert_bytes, get_mem_usage,
27
no_init_or_tensor)
28
-except ImportError:
29
- tensorizer_load_fail = True
28
+except ImportError as e:
29
+ tensorizer_load_fail = e
30
31
__all__ = [
32
'EncryptionParams', 'DecryptionParams', 'TensorDeserializer',
33
'TensorSerializer', 'open_stream', 'convert_bytes', 'get_mem_usage',
34
- 'no_init_or_tensor'
34
+ 'no_init_or_tensor', 'TensorizerConfig'
35
]
36
37
logger = init_logger(__name__)
38
39
40
+@dataclass
41
+class TensorizerConfig:
42
+ tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
43
+ str, bytes, os.PathLike, int]
44
+ vllm_tensorized: bool
45
+ verify_hash: Optional[bool] = False
46
+ num_readers: Optional[int] = 1
47
+ encryption_keyfile: Optional[str] = None
48
+ s3_access_key_id: Optional[str] = None
49
+ s3_secret_access_key: Optional[str] = None
50
+ s3_endpoint: Optional[str] = None
51
+ model_class: Optional[Type[torch.nn.Module]] = None
52
+ hf_config: Optional[PretrainedConfig] = None
53
+ dtype: Optional[Union[str, torch.dtype]] = None
54
+
55
+ def _construct_tensorizer_args(self) -> "TensorizerArgs":
56
+ tensorizer_args = {
57
+ "tensorizer_uri": self.tensorizer_uri,
58
+ "vllm_tensorized": self.vllm_tensorized,
59
+ "verify_hash": self.verify_hash,
60
+ "num_readers": self.num_readers,
61
+ "encryption_keyfile": self.encryption_keyfile,
62
+ "s3_access_key_id": self.s3_access_key_id,
63
+ "s3_secret_access_key": self.s3_secret_access_key,
64
+ "s3_endpoint": self.s3_endpoint,
65
+ }
66
+ return TensorizerArgs(**tensorizer_args)
67
+
68
+ def verify_with_parallel_config(
69
+ self,
70
+ parallel_config: "ParallelConfig",
71
+ ) -> None:
72
+ if (parallel_config.tensor_parallel_size > 1
73
+ and self.tensorizer_uri is not None):
74
+ raise ValueError(
75
+ "Loading to multiple GPUs is not currently supported with "
76
+ "vLLM-serialized models. Please set tensor_parallel_size=1."
77
+ " or use a non-vLLM-serialized model, such as a "
78
+ "serialized Hugging Face `PretrainedModel`.")
79
+
80
+ def verify_with_model_config(self, model_config: "ModelConfig") -> None:
81
+ if (model_config.quantization is not None
82
+ and self.tensorizer_uri is not None):
83
+ logger.warning(
84
+ "Loading a model using Tensorizer with quantization on vLLM"
85
+ " is unstable and may lead to errors.")
86
+
87
+
88
def load_with_tensorizer(tensorizer_config: TensorizerConfig,
89
**extra_kwargs) -> nn.Module:
90
tensorizer = TensorizerAgent(tensorizer_config, **extra_kwargs)
91
return tensorizer.deserialize()
92
93
46
-def tensorizer_warning(message: str):
47
- return warnings.warn(message, category=PerformanceWarning, stacklevel=2)
48
-
49
-
94
def is_vllm_serialized_tensorizer(tensorizer_config: TensorizerConfig) -> bool:
95
if tensorizer_config is None:
96
return False
97
return tensorizer_config.vllm_tensorized
98
99
56
-class ParameterizedLoadFormat(str):
57
- __slots__ = "params"
58
-
59
-
60
-class PerformanceWarning(UserWarning):
61
-
62
- def __str__(self):
63
- return (f"{super().__str__()}"
64
- " (set the VLLM_SILENCE_PERFORMANCE_WARNINGS"
65
- " environment variable to hide this)")
66
-
67
-
68
-if (os.getenv("VLLM_SILENCE_PERFORMANCE_WARNINGS", "").lower()
69
- not in ("", "0", "n", "no", "off", "disable")):
70
- warnings.simplefilter("ignore", category=PerformanceWarning)
71
-
72
-
100
@dataclass
101
class TensorizerArgs:
102
tensorizer_uri: Union[io.BufferedIOBase, io.RawIOBase, typing.BinaryIO,
@@ -219,11 +246,17 @@ class TensorizerAgent:
246
behavior of the TensorDeserializer when loading tensors from a serialized
247
model. For deserializations of HuggingFace models, TensorDeserializer is
248
instead used as an iterator directly in the func hf_model_weights_iterator
222
- in vllm/model_executor/weight_utils.py
249
+ in vllm/model_executor/model_loader/weight_utils.py
250
"""
251
252
def __init__(self, tensorizer_config: TensorizerConfig,
253
linear_method: LinearMethodBase, **extra_kwargs):
254
+ if tensorizer_load_fail is not None:
255
+ raise ImportError(
256
+ "Tensorizer is not installed. Please install tensorizer "
257
+ "to use this feature with `pip install vllm[tensorizer]`."
258
+ ) from tensorizer_load_fail
259
+
260
self.tensorizer_config = tensorizer_config
261
self.tensorizer_args = (
262
self.tensorizer_config._construct_tensorizer_args())
@@ -234,11 +267,6 @@ class TensorizerAgent:
267
self.linear_method = linear_method
268
self.model = self._init_model()
269
237
- if tensorizer_load_fail:
238
- raise ImportError(
239
- "Tensorizer is not installed. Please install tensorizer "
240
- "to use this feature with `pip install vllm[tensorizer]`.")
241
-
270
def _init_model(self):
271
model_args = self.tensorizer_config.hf_config
272
model_args.torch_dtype = self.tensorizer_config.dtype
@@ -313,3 +341,23 @@ class TensorizerAgent:
341
self._check_tensors_on_meta_device()
342
self._resize_lora_embeddings()
343
return self.model.eval()
344
+
345
+
346
+def tensorizer_weights_iterator(
347
+ tensorizer_args: "TensorizerArgs"
348
+) -> Generator[Tuple[str, torch.Tensor], None, None]:
349
+ logger.warning(
350
+ "Deserializing HuggingFace models is not optimized for "
351
+ "loading on vLLM, as tensorizer is forced to load to CPU. "
352
+ "Consider deserializing a vLLM model instead for faster "
353
+ "load times. See the examples/tensorize_vllm_model.py example "
354
+ "script for serializing vLLM models.")
355
+
356
+ deserializer_args = tensorizer_args.deserializer_params
357
+ stream_params = tensorizer_args.stream_params
358
+ stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
359
+ with TensorDeserializer(stream, **deserializer_args,
360
+ device="cpu") as state:
361
+ for name, param in state.items():
362
+ yield name, param
363
+ del state
vllm/model_executor/model_loader/utils.py
new
+40
@@ -0,0 +1,40 @@
1
+"""Utilities for selecting and loading models."""
2
+import contextlib
3
+from typing import Tuple, Type
4
+
5
+import torch
6
+from torch import nn
7
+
8
+from vllm.config import ModelConfig
9
+from vllm.model_executor.models import ModelRegistry
10
+
11
+
12
+@contextlib.contextmanager
13
+def set_default_torch_dtype(dtype: torch.dtype):
14
+ """Sets the default torch dtype to the given dtype."""
15
+ old_dtype = torch.get_default_dtype()
16
+ torch.set_default_dtype(dtype)
17
+ yield
18
+ torch.set_default_dtype(old_dtype)
19
+
20
+
21
+def get_model_architecture(
22
+ model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
23
+ architectures = getattr(model_config.hf_config, "architectures", [])
24
+ # Special handling for quantized Mixtral.
25
+ # FIXME(woosuk): This is a temporary hack.
26
+ if (model_config.quantization is not None
27
+ and "MixtralForCausalLM" in architectures):
28
+ architectures = ["QuantMixtralForCausalLM"]
29
+
30
+ for arch in architectures:
31
+ model_cls = ModelRegistry.load_model_cls(arch)
32
+ if model_cls is not None:
33
+ return (model_cls, arch)
34
+ raise ValueError(
35
+ f"Model architectures {architectures} are not supported for now. "
36
+ f"Supported architectures: {ModelRegistry.get_supported_archs()}")
37
+
38
+
39
+def get_architecture_class_name(model_config: ModelConfig) -> str:
40
+ return get_model_architecture(model_config)[1]
vllm/model_executor/model_loader/weight_utils.py
renamed
+127
-168
@@ -4,8 +4,9 @@ import glob
4
import hashlib
5
import json
6
import os
7
+import tempfile
8
from collections import defaultdict
8
-from typing import Any, Iterable, Iterator, List, Optional, Tuple, Union
9
+from typing import Any, Generator, Iterable, List, Optional, Tuple
10
11
import filelock
12
import huggingface_hub.constants
@@ -15,7 +16,7 @@ from huggingface_hub import HfFileSystem, snapshot_download
16
from safetensors.torch import load_file, safe_open, save_file
17
from tqdm.auto import tqdm
18
18
-from vllm.config import ModelConfig
19
+from vllm.config import LoadConfig, ModelConfig
20
from vllm.logger import init_logger
21
from vllm.model_executor.layers.quantization import (QuantizationConfig,
22
get_quantization_config)
@@ -27,8 +28,7 @@ logger = init_logger(__name__)
28
# can share the same lock without error.
29
# lock files in the temp directory will be automatically deleted when the
30
# system reboots, so users will not complain about annoying lock files
30
-temp_dir = os.environ.get('TMPDIR') or os.environ.get(
31
- 'TEMP') or os.environ.get('TMP') or "/tmp/"
31
+temp_dir = tempfile.gettempdir()
32
33
34
def enable_hf_transfer():
@@ -46,7 +46,7 @@ def enable_hf_transfer():
46
enable_hf_transfer()
47
48
49
-class Disabledtqdm(tqdm):
49
+class DisabledTqdm(tqdm):
50
51
def __init__(self, *args, **kwargs):
52
super().__init__(*args, **kwargs, disable=True)
@@ -114,7 +114,8 @@ def convert_bin_to_safetensor_file(
114
115
116
# TODO(woosuk): Move this to other place.
117
-def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
117
+def get_quant_config(model_config: ModelConfig,
118
+ load_config: LoadConfig) -> QuantizationConfig:
119
quant_cls = get_quantization_config(model_config.quantization)
120
# Read the quantization config from the HF model config, if available.
121
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
@@ -125,12 +126,12 @@ def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
126
is_local = os.path.isdir(model_name_or_path)
127
if not is_local:
128
# Download the config files.
128
- with get_lock(model_name_or_path, model_config.download_dir):
129
+ with get_lock(model_name_or_path, load_config.download_dir):
130
hf_folder = snapshot_download(model_name_or_path,
131
revision=model_config.revision,
132
allow_patterns="*.json",
132
- cache_dir=model_config.download_dir,
133
- tqdm_class=Disabledtqdm)
133
+ cache_dir=load_config.download_dir,
134
+ tqdm_class=DisabledTqdm)
135
else:
136
hf_folder = model_name_or_path
137
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
@@ -153,169 +154,127 @@ def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
154
return quant_cls.from_config(config)
155
156
156
-def prepare_hf_model_weights(
157
- model_name_or_path: str,
158
- cache_dir: Optional[str] = None,
159
- load_format: str = "auto",
160
- fall_back_to_pt: bool = True,
161
- revision: Optional[str] = None,
162
-) -> Tuple[str, List[str], bool]:
163
- # Download model weights from huggingface.
164
- is_local = os.path.isdir(model_name_or_path) \
165
- and load_format != "tensorizer"
166
- use_safetensors = False
167
- # Some quantized models use .pt files for storing the weights.
168
- if load_format == "auto":
169
- allow_patterns = ["*.safetensors", "*.bin"]
170
- elif load_format == "safetensors":
171
- use_safetensors = True
172
- allow_patterns = ["*.safetensors"]
173
- elif load_format == "pt":
174
- allow_patterns = ["*.pt"]
175
- elif load_format == "npcache":
176
- allow_patterns = ["*.bin"]
177
- elif load_format == "tensorizer":
178
- allow_patterns = ["*.tensors"]
179
- else:
180
- raise ValueError(f"Unknown load_format: {load_format}")
181
-
182
- if fall_back_to_pt:
183
- allow_patterns += ["*.pt"]
184
-
185
- if not is_local and load_format != "tensorizer":
186
- # Before we download we look at that is available:
187
- fs = HfFileSystem()
188
- file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
189
-
190
- # depending on what is available we download different things
191
- for pattern in allow_patterns:
192
- matching = fnmatch.filter(file_list, pattern)
193
- if len(matching) > 0:
194
- allow_patterns = [pattern]
195
- break
196
-
197
- logger.info(f"Using model weights format {allow_patterns}")
198
- # Use file lock to prevent multiple processes from
199
- # downloading the same model weights at the same time.
200
- with get_lock(model_name_or_path, cache_dir):
201
- hf_folder = snapshot_download(model_name_or_path,
202
- allow_patterns=allow_patterns,
203
- cache_dir=cache_dir,
204
- tqdm_class=Disabledtqdm,
205
- revision=revision)
206
- else:
207
- hf_folder = model_name_or_path
208
- hf_weights_files: List[str] = []
157
+def download_weights_from_hf(model_name_or_path: str,
158
+ cache_dir: Optional[str],
159
+ allow_patterns: List[str],
160
+ revision: Optional[str] = None) -> str:
161
+ """Download model weights from Hugging Face Hub.
162
+
163
+ Args:
164
+ model_name_or_path (str): The model name or path.
165
+ cache_dir (Optional[str]): The cache directory to store the model
166
+ weights. If None, will use HF defaults.
167
+ allow_patterns (List[str]): The allowed patterns for the
168
+ weight files. Files matched by any of the patterns will be
169
+ downloaded.
170
+ revision (Optional[str]): The revision of the model.
171
+
172
+ Returns:
173
+ str: The path to the downloaded model weights.
174
+ """
175
+ # Before we download we look at that is available:
176
+ fs = HfFileSystem()
177
+ file_list = fs.ls(model_name_or_path, detail=False, revision=revision)
178
+
179
+ # depending on what is available we download different things
180
for pattern in allow_patterns:
210
- hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
211
- if len(hf_weights_files) > 0:
212
- if pattern == "*.safetensors":
213
- use_safetensors = True
181
+ matching = fnmatch.filter(file_list, pattern)
182
+ if len(matching) > 0:
183
+ allow_patterns = [pattern]
184
break
215
- if not use_safetensors:
216
- # Exclude files that are not needed for inference.
217
- # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
218
- blacklist = [
219
- "training_args.bin",
220
- "optimizer.bin",
221
- "optimizer.pt",
222
- "scheduler.pt",
223
- "scaler.pt",
224
- ]
225
- hf_weights_files = [
226
- f for f in hf_weights_files
227
- if not any(f.endswith(x) for x in blacklist)
228
- ]
229
-
230
- if load_format == "tensorizer":
231
- return hf_folder, hf_weights_files, use_safetensors
232
-
233
- if len(hf_weights_files) == 0:
234
- raise RuntimeError(
235
- f"Cannot find any model weights with `{model_name_or_path}`")
236
-
237
- return hf_folder, hf_weights_files, use_safetensors
238
-
239
-
240
-def hf_model_weights_iterator(
241
- model_name_or_path: str,
242
- cache_dir: Optional[str] = None,
243
- load_format: Union[Tuple, str] = "auto",
244
- revision: Optional[str] = None,
245
- fall_back_to_pt: Optional[bool] = True,
246
-) -> Iterator[Tuple[str, torch.Tensor]]:
247
- hf_folder, hf_weights_files, use_safetensors = prepare_hf_model_weights(
248
- model_name_or_path,
249
- cache_dir=cache_dir,
250
- load_format=load_format,
251
- fall_back_to_pt=fall_back_to_pt,
252
- revision=revision)
253
-
254
- if load_format == "npcache":
255
- # Currently np_cache only support *.bin checkpoints
256
- assert use_safetensors is False
257
-
258
- # Convert the model weights from torch tensors to numpy arrays for
259
- # faster loading.
260
- np_folder = os.path.join(hf_folder, "np")
261
- os.makedirs(np_folder, exist_ok=True)
262
- weight_names_file = os.path.join(np_folder, "weight_names.json")
263
- # Use file lock to prevent multiple processes from
264
- # dumping the same model weights to numpy at the same time.
265
- with get_lock(model_name_or_path, cache_dir):
266
- if not os.path.exists(weight_names_file):
267
- weight_names = []
268
- for bin_file in hf_weights_files:
269
- state = torch.load(bin_file, map_location="cpu")
270
- for name, param in state.items():
271
- param_path = os.path.join(np_folder, name)
272
- with open(param_path, "wb") as f:
273
- np.save(f, param.cpu().detach().numpy())
274
- weight_names.append(name)
275
- with open(weight_names_file, "w") as f:
276
- json.dump(weight_names, f)
277
-
278
- with open(weight_names_file, "r") as f:
279
- weight_names = json.load(f)
280
-
281
- for name in weight_names:
282
- param_path = os.path.join(np_folder, name)
283
- with open(param_path, "rb") as f:
284
- param = np.load(f)
285
- yield name, torch.from_numpy(param)
286
- elif load_format == "tensorizer":
287
- from vllm.model_executor.tensorizer_loader import (TensorDeserializer,
288
- open_stream,
289
- tensorizer_warning)
290
- tensorizer_args = load_format.params
291
- tensorizer_warning(
292
- "Deserializing HuggingFace models is not optimized for "
293
- "loading on vLLM, as tensorizer is forced to load to CPU. "
294
- "Consider deserializing a vLLM model instead for faster "
295
- "load times. See the examples/tensorize_vllm_model.py example "
296
- "script for serializing vLLM models.")
297
-
298
- deserializer_args = tensorizer_args.deserializer_params
299
- stream_params = tensorizer_args.stream_params
300
- stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
301
- with TensorDeserializer(stream, **deserializer_args,
302
- device="cpu") as state:
303
- for name, param in state.items():
185
+
186
+ logger.info(f"Using model weights format {allow_patterns}")
187
+ # Use file lock to prevent multiple processes from
188
+ # downloading the same model weights at the same time.
189
+ with get_lock(model_name_or_path, cache_dir):
190
+ hf_folder = snapshot_download(model_name_or_path,
191
+ allow_patterns=allow_patterns,
192
+ cache_dir=cache_dir,
193
+ tqdm_class=DisabledTqdm,
194
+ revision=revision)
195
+ return hf_folder
196
+
197
+
198
+def filter_files_not_needed_for_inference(
199
+ hf_weights_files: List[str]) -> List[str]:
200
+ """
201
+ Exclude files that are not needed for inference.
202
+
203
+ See https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
204
+ """
205
+ blacklist = [
206
+ "training_args.bin",
207
+ "optimizer.bin",
208
+ "optimizer.pt",
209
+ "scheduler.pt",
210
+ "scaler.pt",
211
+ ]
212
+ hf_weights_files = [
213
+ f for f in hf_weights_files
214
+ if not any(f.endswith(x) for x in blacklist)
215
+ ]
216
+ return hf_weights_files
217
+
218
+
219
+def np_cache_weights_iterator(
220
+ model_name_or_path: str, cache_dir: Optional[str], hf_folder: str,
221
+ hf_weights_files: List[str]
222
+) -> Generator[Tuple[str, torch.Tensor], None, None]:
223
+ """Iterate over the weights in the model np files.
224
+
225
+ Will dump the model weights to numpy files if they are not already dumped.
226
+ """
227
+ # Convert the model weights from torch tensors to numpy arrays for
228
+ # faster loading.
229
+ np_folder = os.path.join(hf_folder, "np")
230
+ os.makedirs(np_folder, exist_ok=True)
231
+ weight_names_file = os.path.join(np_folder, "weight_names.json")
232
+ # Use file lock to prevent multiple processes from
233
+ # dumping the same model weights to numpy at the same time.
234
+ with get_lock(model_name_or_path, cache_dir):
235
+ if not os.path.exists(weight_names_file):
236
+ weight_names = []
237
+ for bin_file in hf_weights_files:
238
+ state = torch.load(bin_file, map_location="cpu")
239
+ for name, param in state.items():
240
+ param_path = os.path.join(np_folder, name)
241
+ with open(param_path, "wb") as f:
242
+ np.save(f, param.cpu().detach().numpy())
243
+ weight_names.append(name)
244
+ with open(weight_names_file, "w") as f:
245
+ json.dump(weight_names, f)
246
+
247
+ with open(weight_names_file, "r") as f:
248
+ weight_names = json.load(f)
249
+
250
+ for name in weight_names:
251
+ param_path = os.path.join(np_folder, name)
252
+ with open(param_path, "rb") as f:
253
+ param = np.load(f)
254
+ yield name, torch.from_numpy(param)
255
+
256
+
257
+def safetensors_weights_iterator(
258
+ hf_weights_files: List[str]
259
+) -> Generator[Tuple[str, torch.Tensor], None, None]:
260
+ """Iterate over the weights in the model safetensor files."""
261
+ for st_file in hf_weights_files:
262
+ with safe_open(st_file, framework="pt") as f:
263
+ for name in f.keys(): # noqa: SIM118
264
+ param = f.get_tensor(name)
265
yield name, param
266
+
267
+
268
+def pt_weights_iterator(
269
+ hf_weights_files: List[str]
270
+) -> Generator[Tuple[str, torch.Tensor], None, None]:
271
+ """Iterate over the weights in the model bin/pt files."""
272
+ for bin_file in hf_weights_files:
273
+ state = torch.load(bin_file, map_location="cpu")
274
+ for name, param in state.items():
275
+ yield name, param
276
del state
306
- elif use_safetensors:
307
- for st_file in hf_weights_files:
308
- with safe_open(st_file, framework="pt") as f:
309
- for name in f.keys(): # noqa: SIM118
310
- param = f.get_tensor(name)
311
- yield name, param
312
- else:
313
- for bin_file in hf_weights_files:
314
- state = torch.load(bin_file, map_location="cpu")
315
- for name, param in state.items():
316
- yield name, param
317
- del state
318
- torch.cuda.empty_cache()
277
+ torch.cuda.empty_cache()
278
279
280
def kv_cache_scales_loader(
vllm/model_executor/models/baichuan.py
+4
-10
@@ -19,7 +19,7 @@
19
# limitations under the License.
20
"""Inference-only BaiChuan model compatible with HuggingFace weights."""
21
import math
22
-from typing import List, Optional, Tuple
22
+from typing import Iterable, List, Optional, Tuple
23
24
import torch
25
from torch import nn
@@ -40,9 +40,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
43
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
-from vllm.model_executor.weight_utils import (default_weight_loader,
45
- hf_model_weights_iterator)
45
from vllm.sequence import SamplerOutput
46
47
@@ -340,19 +339,14 @@ class BaiChuanBaseForCausalLM(nn.Module):
339
next_tokens = self.sampler(logits, sampling_metadata)
340
return next_tokens
341
343
- def load_weights(self,
344
- model_name_or_path: str,
345
- cache_dir: Optional[str] = None,
346
- load_format: str = "auto",
347
- revision: Optional[str] = None):
342
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
343
stacked_params_mapping = [
344
# (param_name, shard_name, shard_id)
345
("gate_up_proj", "gate_proj", 0),
346
("gate_up_proj", "up_proj", 1),
347
]
348
params_dict = dict(self.named_parameters())
354
- for name, loaded_weight in hf_model_weights_iterator(
355
- model_name_or_path, cache_dir, load_format, revision):
349
+ for name, loaded_weight in weights:
350
if "rotary_emb.inv_freq" in name:
351
continue
352
if name == "lm_head.weight":
vllm/model_executor/models/bloom.py
+4
-10
@@ -17,7 +17,7 @@
17
# limitations under the License.
18
"""Inference-only BLOOM model compatible with HuggingFace weights."""
19
import math
20
-from typing import List, Optional
20
+from typing import Iterable, List, Optional, Tuple
21
22
import torch
23
from torch import nn
@@ -35,9 +35,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
38
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
-from vllm.model_executor.weight_utils import (default_weight_loader,
40
- hf_model_weights_iterator)
40
from vllm.sequence import SamplerOutput
41
42
@@ -298,14 +297,9 @@ class BloomForCausalLM(nn.Module):
297
next_tokens = self.sampler(logits, sampling_metadata)
298
return next_tokens
299
301
- def load_weights(self,
302
- model_name_or_path: str,
303
- cache_dir: Optional[str] = None,
304
- load_format: str = "auto",
305
- revision: Optional[str] = None):
300
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
301
params_dict = dict(self.named_parameters(remove_duplicate=False))
307
- for name, loaded_weight in hf_model_weights_iterator(
308
- model_name_or_path, cache_dir, load_format, revision):
302
+ for name, loaded_weight in weights:
303
if name == "lm_head.weight":
304
continue
305
if not name.startswith("transformer."):
vllm/model_executor/models/chatglm.py
+4
-10
@@ -2,7 +2,7 @@
2
# Adapted from
3
# https://github.com/THUDM/ChatGLM2-6B
4
"""Inference-only ChatGLM model compatible with THUDM weights."""
5
-from typing import List, Optional
5
+from typing import Iterable, List, Optional, Tuple
6
7
import torch
8
from torch import nn
@@ -22,9 +22,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22
from vllm.model_executor.layers.sampler import Sampler
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
ParallelLMHead, VocabParallelEmbedding)
25
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
-from vllm.model_executor.weight_utils import (default_weight_loader,
27
- hf_model_weights_iterator)
27
from vllm.sequence import SamplerOutput
28
from vllm.transformers_utils.configs import ChatGLMConfig
29
@@ -370,14 +369,9 @@ class ChatGLMForCausalLM(nn.Module):
369
next_tokens = self.sampler(logits, sampling_metadata)
370
return next_tokens
371
373
- def load_weights(self,
374
- model_name_or_path: str,
375
- cache_dir: Optional[str] = None,
376
- load_format: str = "auto",
377
- revision: Optional[str] = None):
372
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
373
params_dict = dict(self.named_parameters(remove_duplicate=False))
379
- for name, loaded_weight in hf_model_weights_iterator(
380
- model_name_or_path, cache_dir, load_format, revision):
374
+ for name, loaded_weight in weights:
375
if "rotary_pos_emb.inv_freq" in name:
376
continue
377
if "word_embeddings" in name:
vllm/model_executor/models/commandr.py
+4
-12
@@ -20,7 +20,7 @@
20
21
# This file is based on the LLama model definition file in transformers
22
"""PyTorch Cohere model."""
23
-from typing import List, Optional, Tuple
23
+from typing import Iterable, List, Optional, Tuple
24
25
import torch
26
import torch.utils.checkpoint
@@ -41,10 +41,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
VocabParallelEmbedding)
44
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.model_executor.utils import set_weight_attrs
46
-from vllm.model_executor.weight_utils import (default_weight_loader,
47
- hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
48
49
@@ -335,13 +334,7 @@ class CohereForCausalLM(nn.Module):
334
next_tokens = self.sampler(logits, sampling_metadata)
335
return next_tokens
336
338
- def load_weights(
339
- self,
340
- model_name_or_path: str,
341
- cache_dir: Optional[str] = None,
342
- load_format: str = "auto",
343
- revision: Optional[str] = None,
344
- ):
337
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
338
stacked_params_mapping = [
339
# (param_name, shard_name, shard_id)
340
("qkv_proj", "q_proj", "q"),
@@ -352,8 +345,7 @@ class CohereForCausalLM(nn.Module):
345
]
346
params_dict = dict(self.named_parameters())
347
loaded_params = set()
355
- for name, loaded_weight in hf_model_weights_iterator(
356
- model_name_or_path, cache_dir, load_format, revision):
348
+ for name, loaded_weight in weights:
349
for param_name, shard_name, shard_id in stacked_params_mapping:
350
if shard_name not in name:
351
continue
vllm/model_executor/models/dbrx.py
+4
-12
@@ -1,5 +1,5 @@
1
# coding=utf-8
2
-from typing import List, Optional
2
+from typing import Iterable, List, Optional, Tuple
3
4
import torch
5
import torch.nn as nn
@@ -18,10 +18,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
21
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.model_executor.utils import set_weight_attrs
23
-from vllm.model_executor.weight_utils import (default_weight_loader,
24
- hf_model_weights_iterator)
24
from vllm.sequence import SamplerOutput
25
from vllm.transformers_utils.configs.dbrx import DbrxConfig
26
@@ -391,20 +390,13 @@ class DbrxForCausalLM(nn.Module):
390
next_tokens = self.sampler(logits, sampling_metadata)
391
return next_tokens
392
394
- def load_weights(
395
- self,
396
- model_name_or_path: str,
397
- cache_dir: Optional[str] = None,
398
- load_format: str = "auto",
399
- revision: Optional[str] = None,
400
- ):
393
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
394
expert_params_mapping = [(
395
"ws" if weight_name in ["w1", "v1"] else "w2s",
396
f"experts.mlp.{weight_name}",
397
) for weight_name in ["w1", "v1", "w2"]]
398
params_dict = dict(self.named_parameters(remove_duplicate=False))
406
- for name, loaded_weight in hf_model_weights_iterator(
407
- model_name_or_path, cache_dir, load_format, revision):
399
+ for name, loaded_weight in weights:
400
for param_name, weight_name in expert_params_mapping:
401
if weight_name not in name:
402
continue
vllm/model_executor/models/decilm.py
+4
-10
@@ -23,16 +23,15 @@
23
# limitations under the License.
24
"""Inference-only DeciLM model compatible with HuggingFace weights."""
25
26
-from typing import Optional
26
+from typing import Iterable, Optional, Tuple
27
28
import torch
29
from transformers import PretrainedConfig
30
31
from vllm.config import LoRAConfig
32
from vllm.model_executor.layers.linear import LinearMethodBase
33
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
from vllm.model_executor.models.llama import LlamaForCausalLM
34
-from vllm.model_executor.weight_utils import (default_weight_loader,
35
- hf_model_weights_iterator)
35
36
37
class DeciLMForCausalLM(LlamaForCausalLM):
@@ -65,11 +64,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
64
linear_method=linear_method,
65
lora_config=lora_config)
66
68
- def load_weights(self,
69
- model_name_or_path: str,
70
- cache_dir: Optional[str] = None,
71
- load_format: str = "auto",
72
- revision: Optional[str] = None):
67
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
68
stacked_params_mapping = [
69
# (param_name, shard_name, shard_id)
70
("qkv_proj", "q_proj", "q"),
@@ -79,8 +74,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
74
("gate_up_proj", "up_proj", 1),
75
]
76
params_dict = dict(self.named_parameters())
82
- for name, loaded_weight in hf_model_weights_iterator(
83
- model_name_or_path, cache_dir, load_format, revision):
77
+ for name, loaded_weight in weights:
78
if "rotary_emb.inv_freq" in name:
79
continue
80
vllm/model_executor/models/deepseek.py
+6
-14
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Deepseek model."""
24
-from typing import Any, Dict, List, Optional
24
+from typing import Any, Dict, Iterable, List, Optional, Tuple
25
26
import torch
27
from torch import nn
@@ -44,9 +44,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
44
from vllm.model_executor.layers.sampler import Sampler
45
from vllm.model_executor.layers.vocab_parallel_embedding import (
46
ParallelLMHead, VocabParallelEmbedding)
47
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
-from vllm.model_executor.weight_utils import (default_weight_loader,
49
- hf_model_weights_iterator)
49
from vllm.sequence import SamplerOutput
50
51
@@ -316,6 +315,8 @@ class DeepseekDecoderLayer(nn.Module):
315
316
class DeepseekModel(nn.Module):
317
318
+ fall_back_to_pt_during_load = False
319
+
320
def __init__(
321
self,
322
config: PretrainedConfig,
@@ -395,11 +396,7 @@ class DeepseekForCausalLM(nn.Module):
396
next_tokens = self.sampler(logits, sampling_metadata)
397
return next_tokens
398
398
- def load_weights(self,
399
- model_name_or_path: str,
400
- cache_dir: Optional[str] = None,
401
- load_format: str = "auto",
402
- revision: Optional[str] = None):
399
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
400
stacked_params_mapping = [
401
# (param_name, shard_name, shard_id)
402
("qkv_proj", "q_proj", "q"),
@@ -410,12 +407,7 @@ class DeepseekForCausalLM(nn.Module):
407
]
408
409
params_dict = dict(self.named_parameters())
413
- for name, loaded_weight in hf_model_weights_iterator(
414
- model_name_or_path,
415
- cache_dir,
416
- load_format,
417
- revision,
418
- fall_back_to_pt=False):
410
+ for name, loaded_weight in weights:
411
if "rotary_emb.inv_freq" in name:
412
continue
413
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/falcon.py
+4
-10
@@ -19,7 +19,7 @@
19
"""PyTorch Falcon model."""
20
21
import math
22
-from typing import List, Optional, Union
22
+from typing import Iterable, List, Optional, Tuple, Union
23
24
import torch
25
from torch import nn
@@ -40,9 +40,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
VocabParallelEmbedding)
43
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
-from vllm.model_executor.weight_utils import (default_weight_loader,
45
- hf_model_weights_iterator)
45
from vllm.sequence import SamplerOutput
46
from vllm.transformers_utils.configs import RWConfig
47
@@ -399,11 +398,7 @@ class FalconForCausalLM(nn.Module):
398
next_tokens = self.sampler(logits, sampling_metadata)
399
return next_tokens
400
402
- def load_weights(self,
403
- model_name_or_path: str,
404
- cache_dir: Optional[str] = None,
405
- load_format: str = "auto",
406
- revision: Optional[str] = None):
401
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
402
total_num_heads = self.config.num_attention_heads
403
if self.config.new_decoder_architecture:
404
total_num_kv_heads = self.config.num_kv_heads
@@ -413,8 +408,7 @@ class FalconForCausalLM(nn.Module):
408
total_num_kv_heads = total_num_heads
409
num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
410
params_dict = dict(self.named_parameters(remove_duplicate=False))
416
- for name, loaded_weight in hf_model_weights_iterator(
417
- model_name_or_path, cache_dir, load_format, revision):
411
+ for name, loaded_weight in weights:
412
if name == "lm_head.weight":
413
# Falcon uses tied embeddings.
414
continue
vllm/model_executor/models/gemma.py
+4
-10
@@ -15,7 +15,7 @@
15
# limitations under the License.
16
"""Inference-only Gemma model compatible with HuggingFace weights."""
17
from functools import lru_cache
18
-from typing import List, Optional, Tuple
18
+from typing import Iterable, List, Optional, Tuple
19
20
import torch
21
from torch import nn
@@ -36,9 +36,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
39
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
-from vllm.model_executor.weight_utils import (default_weight_loader,
41
- hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
43
logger = init_logger(__name__)
@@ -346,11 +345,7 @@ class GemmaForCausalLM(nn.Module):
345
next_tokens = self.sampler(logits, sampling_metadata)
346
return next_tokens
347
349
- def load_weights(self,
350
- model_name_or_path: str,
351
- cache_dir: Optional[str] = None,
352
- load_format: str = "auto",
353
- revision: Optional[str] = None):
348
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
349
stacked_params_mapping = [
350
# (param_name, shard_name, shard_id)
351
("qkv_proj", "q_proj", "q"),
@@ -361,8 +356,7 @@ class GemmaForCausalLM(nn.Module):
356
]
357
params_dict = dict(self.named_parameters())
358
loaded_params = set()
364
- for name, loaded_weight in hf_model_weights_iterator(
365
- model_name_or_path, cache_dir, load_format, revision):
359
+ for name, loaded_weight in weights:
360
for (param_name, shard_name, shard_id) in stacked_params_mapping:
361
if shard_name not in name:
362
continue
vllm/model_executor/models/gpt2.py
+4
-10
@@ -17,7 +17,7 @@
17
# See the License for the specific language governing permissions and
18
# limitations under the License.
19
"""Inference-only GPT-2 model compatible with HuggingFace weights."""
20
-from typing import List, Optional
20
+from typing import Iterable, List, Optional, Tuple
21
22
import torch
23
from torch import nn
@@ -34,9 +34,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding)
37
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
-from vllm.model_executor.weight_utils import (default_weight_loader,
39
- hf_model_weights_iterator)
39
from vllm.sequence import SamplerOutput
40
41
@@ -239,14 +238,9 @@ class GPT2LMHeadModel(nn.Module):
238
next_tokens = self.sampler(logits, sampling_metadata)
239
return next_tokens
240
242
- def load_weights(self,
243
- model_name_or_path: str,
244
- cache_dir: Optional[str] = None,
245
- load_format: str = "auto",
246
- revision: Optional[str] = None):
241
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
242
params_dict = dict(self.named_parameters(remove_duplicate=False))
248
- for name, loaded_weight in hf_model_weights_iterator(
249
- model_name_or_path, cache_dir, load_format, revision):
243
+ for name, loaded_weight in weights:
244
if "lm_head.weight" in name:
245
# GPT-2 ties the weights of the embedding layer and the final
246
# linear layer.
vllm/model_executor/models/gpt_bigcode.py
+4
-10
@@ -18,7 +18,7 @@
18
# See the License for the specific language governing permissions and
19
# limitations under the License.
20
"""Inference-only GPTBigCode model compatible with HuggingFace weights."""
21
-from typing import List, Optional
21
+from typing import Iterable, List, Optional, Tuple
22
23
import torch
24
from torch import nn
@@ -35,9 +35,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
38
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
-from vllm.model_executor.weight_utils import (default_weight_loader,
40
- hf_model_weights_iterator)
40
from vllm.sequence import SamplerOutput
41
42
@@ -260,14 +259,9 @@ class GPTBigCodeForCausalLM(nn.Module):
259
next_tokens = self.sampler(logits, sampling_metadata)
260
return next_tokens
261
263
- def load_weights(self,
264
- model_name_or_path: str,
265
- cache_dir: Optional[str] = None,
266
- load_format: str = "auto",
267
- revision: Optional[str] = None):
262
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
263
params_dict = dict(self.named_parameters(remove_duplicate=False))
269
- for name, loaded_weight in hf_model_weights_iterator(
270
- model_name_or_path, cache_dir, load_format, revision):
264
+ for name, loaded_weight in weights:
265
if "lm_head.weight" in name:
266
continue
267
if ".attn.bias" in name:
vllm/model_executor/models/gpt_j.py
+4
-10
@@ -16,7 +16,7 @@
16
# See the License for the specific language governing permissions and
17
# limitations under the License.
18
"""Inference-only GPT-J model compatible with HuggingFace weights."""
19
-from typing import List, Optional
19
+from typing import Iterable, List, Optional, Tuple
20
21
import torch
22
from torch import nn
@@ -34,9 +34,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
ParallelLMHead, VocabParallelEmbedding)
37
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
-from vllm.model_executor.weight_utils import (default_weight_loader,
39
- hf_model_weights_iterator)
39
from vllm.sequence import SamplerOutput
40
41
@@ -248,11 +247,7 @@ class GPTJForCausalLM(nn.Module):
247
next_tokens = self.sampler(logits, sampling_metadata)
248
return next_tokens
249
251
- def load_weights(self,
252
- model_name_or_path: str,
253
- cache_dir: Optional[str] = None,
254
- load_format: str = "auto",
255
- revision: Optional[str] = None):
250
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
251
stacked_params_mapping = [
252
# (param_name, shard_name, shard_id)
253
("qkv_proj", "q_proj", "q"),
@@ -262,8 +257,7 @@ class GPTJForCausalLM(nn.Module):
257
("gate_up_proj", "up_proj", 1),
258
]
259
params_dict = dict(self.named_parameters())
265
- for name, loaded_weight in hf_model_weights_iterator(
266
- model_name_or_path, cache_dir, load_format, revision):
260
+ for name, loaded_weight in weights:
261
if "attn.bias" in name or "attn.masked_bias" in name:
262
continue
263
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/gpt_neox.py
+4
-10
@@ -16,7 +16,7 @@
16
# See the License for the specific language governing permissions and
17
# limitations under the License.
18
"""Inference-only GPT-NeoX model compatible with HuggingFace weights."""
19
-from typing import List, Optional
19
+from typing import Iterable, List, Optional, Tuple
20
21
import torch
22
from torch import nn
@@ -34,9 +34,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
ParallelLMHead, VocabParallelEmbedding)
37
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
-from vllm.model_executor.weight_utils import (default_weight_loader,
39
- hf_model_weights_iterator)
39
from vllm.sequence import SamplerOutput
40
41
@@ -262,14 +261,9 @@ class GPTNeoXForCausalLM(nn.Module):
261
next_tokens = self.sampler(logits, sampling_metadata)
262
return next_tokens
263
265
- def load_weights(self,
266
- model_name_or_path: str,
267
- cache_dir: Optional[str] = None,
268
- load_format: str = "auto",
269
- revision: Optional[str] = None):
264
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
265
params_dict = dict(self.named_parameters())
271
- for name, loaded_weight in hf_model_weights_iterator(
272
- model_name_or_path, cache_dir, load_format, revision):
266
+ for name, loaded_weight in weights:
267
if ("attention.bias" in name or "attention.masked_bias" in name
268
or "rotary_emb.inv_freq" in name):
269
continue
vllm/model_executor/models/internlm2.py
+4
-10
@@ -1,5 +1,5 @@
1
# -*- coding: utf-8 -*-
2
-from typing import Any, Dict, List, Optional, Tuple
2
+from typing import Any, Dict, Iterable, List, Optional, Tuple
3
4
import torch
5
from torch import nn
@@ -18,9 +18,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
ParallelLMHead, VocabParallelEmbedding)
21
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
-from vllm.model_executor.weight_utils import (default_weight_loader,
23
- hf_model_weights_iterator)
23
from vllm.sequence import SamplerOutput
24
25
@@ -274,19 +273,14 @@ class InternLM2ForCausalLM(nn.Module):
273
next_tokens = self.sampler(logits, sampling_metadata)
274
return next_tokens
275
277
- def load_weights(self,
278
- model_name_or_path: str,
279
- cache_dir: Optional[str] = None,
280
- load_format: str = "auto",
281
- revision: Optional[str] = None):
276
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
277
stacked_params_mapping = [
278
# (param_name, shard_name, shard_id)
279
("gate_up_proj", "w1", 0),
280
("gate_up_proj", "w3", 1),
281
]
282
params_dict = dict(self.named_parameters())
288
- for name, loaded_weight in hf_model_weights_iterator(
289
- model_name_or_path, cache_dir, load_format, revision):
283
+ for name, loaded_weight in weights:
284
if "rotary_emb.inv_freq" in name:
285
continue
286
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/jais.py
+4
-12
@@ -20,7 +20,7 @@
20
"""Inference-only Jais model compatible with HuggingFace weights."""
21
22
import math
23
-from typing import List, Optional
23
+from typing import Iterable, List, Optional, Tuple
24
25
import torch
26
from torch import nn
@@ -36,9 +36,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
39
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
-from vllm.model_executor.weight_utils import (default_weight_loader,
41
- hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
from vllm.transformers_utils.configs import JAISConfig
43
@@ -303,16 +302,9 @@ class JAISLMHeadModel(nn.Module):
302
next_tokens = self.sampler(logits, sampling_metadata)
303
return next_tokens
304
306
- def load_weights(
307
- self,
308
- model_name_or_path: str,
309
- cache_dir: Optional[str] = None,
310
- load_format: str = "auto",
311
- revision: Optional[str] = None,
312
- ):
305
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
306
params_dict = dict(self.named_parameters(remove_duplicate=False))
314
- for name, loaded_weight in hf_model_weights_iterator(
315
- model_name_or_path, cache_dir, load_format, revision):
307
+ for name, loaded_weight in weights:
308
if "lm_head.weight" in name:
309
# GPT-2 ties the weights of the embedding layer and the final
310
# linear layer.
vllm/model_executor/models/llama.py
+5
-11
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only LLaMA model compatible with HuggingFace weights."""
24
-from typing import Any, Dict, List, Optional, Tuple
24
+from typing import Any, Dict, Iterable, List, Optional, Tuple
25
26
import torch
27
from torch import nn
@@ -42,10 +42,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
+from vllm.model_executor.model_loader.weight_utils import (
46
+ default_weight_loader, kv_cache_scales_loader)
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.model_executor.weight_utils import (default_weight_loader,
47
- hf_model_weights_iterator,
48
- kv_cache_scales_loader)
48
from vllm.sequence import SamplerOutput
49
from vllm.utils import is_hip
50
@@ -376,11 +375,7 @@ class LlamaForCausalLM(nn.Module):
375
next_tokens = self.sampler(logits, sampling_metadata)
376
return next_tokens
377
379
- def load_weights(self,
380
- model_name_or_path: str,
381
- cache_dir: Optional[str] = None,
382
- load_format: str = "auto",
383
- revision: Optional[str] = None):
378
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
379
stacked_params_mapping = [
380
# (param_name, shard_name, shard_id)
381
("qkv_proj", "q_proj", "q"),
@@ -390,8 +385,7 @@ class LlamaForCausalLM(nn.Module):
385
("gate_up_proj", "up_proj", 1),
386
]
387
params_dict = dict(self.named_parameters())
393
- for name, loaded_weight in hf_model_weights_iterator(
394
- model_name_or_path, cache_dir, load_format, revision):
388
+ for name, loaded_weight in weights:
389
if "rotary_emb.inv_freq" in name:
390
continue
391
if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/llava.py
+4
-10
@@ -1,4 +1,4 @@
1
-from typing import List, Optional
1
+from typing import Iterable, List, Optional, Tuple
2
3
import torch
4
from torch import nn
@@ -13,10 +13,9 @@ from vllm.model_executor.layers.linear import LinearMethodBase
13
from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
from vllm.model_executor.layers.sampler import Sampler
15
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
16
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.model_executor.models.llama import LlamaModel
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
-from vllm.model_executor.weight_utils import (default_weight_loader,
19
- hf_model_weights_iterator)
19
from vllm.sequence import SamplerOutput
20
21
_KEYS_TO_MODIFY_MAPPING = {
@@ -198,11 +197,7 @@ class LlavaForConditionalGeneration(nn.Module):
197
next_tokens = self.sampler(logits, sampling_metadata)
198
return next_tokens
199
201
- def load_weights(self,
202
- model_name_or_path: str,
203
- cache_dir: Optional[str] = None,
204
- load_format: str = "auto",
205
- revision: Optional[str] = None):
200
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
201
# only doing this for language model part for now.
202
stacked_params_mapping = [
203
# (param_name, shard_name, shard_id)
@@ -213,8 +208,7 @@ class LlavaForConditionalGeneration(nn.Module):
208
("gate_up_proj", "up_proj", 1),
209
]
210
params_dict = dict(self.named_parameters())
216
- for name, loaded_weight in hf_model_weights_iterator(
217
- model_name_or_path, cache_dir, load_format, revision):
211
+ for name, loaded_weight in weights:
212
if "rotary_emb.inv_freq" in name:
213
continue
214
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
vllm/model_executor/models/minicpm.py
+4
-10
@@ -22,7 +22,7 @@
22
# limitations under the License.
23
"""Inference-only MiniCPM model compatible with HuggingFace weights."""
24
import math
25
-from typing import Any, Dict, List, Optional, Tuple
25
+from typing import Any, Dict, Iterable, List, Optional, Tuple
26
27
import torch
28
from torch import nn
@@ -45,10 +45,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
45
from vllm.model_executor.layers.sampler import Sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.model_executor.utils import set_weight_attrs
50
-from vllm.model_executor.weight_utils import (default_weight_loader,
51
- hf_model_weights_iterator)
51
from vllm.sequence import SamplerOutput
52
53
@@ -472,11 +471,7 @@ class MiniCPMForCausalLM(nn.Module):
471
next_tokens = self.sampler(logits, sampling_metadata)
472
return next_tokens
473
475
- def load_weights(self,
476
- model_name_or_path: str,
477
- cache_dir: Optional[str] = None,
478
- load_format: str = "auto",
479
- revision: Optional[str] = None):
474
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
475
stacked_params_mapping = [
476
# (param_name, shard_name, shard_id)
477
("qkv_proj", "q_proj", "q"),
@@ -493,8 +488,7 @@ class MiniCPMForCausalLM(nn.Module):
488
for weight_name in ["w1", "w2", "w3"]
489
]
490
params_dict = dict(self.named_parameters())
496
- for name, loaded_weight in hf_model_weights_iterator(
497
- model_name_or_path, cache_dir, load_format, revision):
491
+ for name, loaded_weight in weights:
492
if "rotary_emb.inv_freq" in name:
493
continue
494
if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/mixtral.py
+6
-14
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Mixtral model."""
24
-from typing import List, Optional
24
+from typing import Iterable, List, Optional, Tuple
25
26
import torch
27
from torch import nn
@@ -43,10 +43,9 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
46
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.model_executor.utils import set_weight_attrs
48
-from vllm.model_executor.weight_utils import (default_weight_loader,
49
- hf_model_weights_iterator)
49
from vllm.sequence import SamplerOutput
50
51
@@ -319,6 +318,8 @@ class MixtralModel(nn.Module):
318
319
320
class MixtralForCausalLM(nn.Module):
321
+ fall_back_to_pt_during_load = False
322
+
323
packed_modules_mapping = {
324
"qkv_proj": [
325
"q_proj",
@@ -393,11 +394,7 @@ class MixtralForCausalLM(nn.Module):
394
next_tokens = self.sampler(logits, sampling_metadata)
395
return next_tokens
396
396
- def load_weights(self,
397
- model_name_or_path: str,
398
- cache_dir: Optional[str] = None,
399
- load_format: str = "auto",
400
- revision: Optional[str] = None):
397
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
398
stacked_params_mapping = [
399
# (param_name, shard_name, shard_id)
400
("qkv_proj", "q_proj", "q"),
@@ -414,12 +411,7 @@ class MixtralForCausalLM(nn.Module):
411
]
412
413
params_dict = dict(self.named_parameters())
417
- for name, loaded_weight in hf_model_weights_iterator(
418
- model_name_or_path,
419
- cache_dir,
420
- load_format,
421
- revision,
422
- fall_back_to_pt=False):
414
+ for name, loaded_weight in weights:
415
if "rotary_emb.inv_freq" in name:
416
continue
417
vllm/model_executor/models/mixtral_quant.py
+5
-14
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Mixtral model."""
24
-from typing import List, Optional
24
+from typing import Iterable, List, Optional, Tuple
25
26
import numpy as np
27
import torch
@@ -43,9 +43,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
ParallelLMHead, VocabParallelEmbedding)
46
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
-from vllm.model_executor.weight_utils import (default_weight_loader,
48
- hf_model_weights_iterator)
48
from vllm.sequence import SamplerOutput
49
50
@@ -327,6 +326,7 @@ class MixtralModel(nn.Module):
326
327
328
class MixtralForCausalLM(nn.Module):
329
+ fall_back_to_pt_during_load = False
330
331
def __init__(
332
self,
@@ -366,11 +366,7 @@ class MixtralForCausalLM(nn.Module):
366
next_tokens = self.sampler(logits, sampling_metadata)
367
return next_tokens
368
369
- def load_weights(self,
370
- model_name_or_path: str,
371
- cache_dir: Optional[str] = None,
372
- load_format: str = "auto",
373
- revision: Optional[str] = None):
369
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
370
stacked_params_mapping = [
371
# (param_name, shard_name, shard_id)
372
("qkv_proj", "q_proj", "q"),
@@ -379,12 +375,7 @@ class MixtralForCausalLM(nn.Module):
375
]
376
377
params_dict = dict(self.named_parameters())
382
- for name, loaded_weight in hf_model_weights_iterator(
383
- model_name_or_path,
384
- cache_dir,
385
- load_format,
386
- revision,
387
- fall_back_to_pt=False):
378
+ for name, loaded_weight in weights:
379
if "rotary_emb.inv_freq" in name:
380
continue
381
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/mpt.py
+4
-10
@@ -1,7 +1,7 @@
1
# coding=utf-8
2
# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
3
import math
4
-from typing import List, Optional
4
+from typing import Iterable, List, Optional, Tuple
5
6
import torch
7
import torch.nn as nn
@@ -18,9 +18,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
VocabParallelEmbedding)
21
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
-from vllm.model_executor.weight_utils import (default_weight_loader,
23
- hf_model_weights_iterator)
23
from vllm.sequence import SamplerOutput
24
from vllm.transformers_utils.configs.mpt import MPTConfig
25
@@ -284,14 +283,9 @@ class MPTForCausalLM(nn.Module):
283
next_tokens = self.sampler(logits, sampling_metadata)
284
return next_tokens
285
287
- def load_weights(self,
288
- model_name_or_path: str,
289
- cache_dir: Optional[str] = None,
290
- load_format: str = "auto",
291
- revision: Optional[str] = None):
286
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
287
params_dict = dict(self.named_parameters(remove_duplicate=False))
293
- for name, loaded_weight in hf_model_weights_iterator(
294
- model_name_or_path, cache_dir, load_format, revision):
288
+ for name, loaded_weight in weights:
289
# Skip loading extra bias for GPTQ models.
290
if name.endswith(".bias") and name not in params_dict:
291
continue
vllm/model_executor/models/olmo.py
+4
-12
@@ -36,7 +36,7 @@
36
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38
"""Inference-only OLMo model compatible with HuggingFace weights."""
39
-from typing import List, Optional, Tuple
39
+from typing import Iterable, List, Optional, Tuple
40
41
import torch
42
# this model must need this dependency
@@ -56,9 +56,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
56
from vllm.model_executor.layers.sampler import Sampler
57
from vllm.model_executor.layers.vocab_parallel_embedding import (
58
VocabParallelEmbedding)
59
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
60
from vllm.model_executor.sampling_metadata import SamplingMetadata
60
-from vllm.model_executor.weight_utils import (default_weight_loader,
61
- hf_model_weights_iterator)
61
from vllm.sequence import SamplerOutput
62
63
@@ -348,16 +347,9 @@ class OLMoForCausalLM(nn.Module):
347
next_tokens = self.sampler(logits, sampling_metadata)
348
return next_tokens
349
351
- def load_weights(
352
- self,
353
- model_name_or_path: str,
354
- cache_dir: Optional[str] = None,
355
- load_format: str = "auto",
356
- revision: Optional[str] = None,
357
- ):
350
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
351
params_dict = dict(self.named_parameters(remove_duplicate=False))
359
- for name, loaded_weight in hf_model_weights_iterator(
360
- model_name_or_path, cache_dir, load_format, revision):
352
+ for name, loaded_weight in weights:
353
# attention
354
if ".att" in name:
355
name = name.replace(".att", ".attn.att")
vllm/model_executor/models/opt.py
+4
-10
@@ -17,7 +17,7 @@
17
# See the License for the specific language governing permissions and
18
# limitations under the License.
19
"""Inference-only OPT model compatible with HuggingFace weights."""
20
-from typing import List, Optional
20
+from typing import Iterable, List, Optional, Tuple
21
22
import torch
23
from torch import nn
@@ -35,9 +35,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
38
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
-from vllm.model_executor.weight_utils import (default_weight_loader,
40
- hf_model_weights_iterator)
40
from vllm.sequence import SamplerOutput
41
42
@@ -315,11 +314,7 @@ class OPTForCausalLM(nn.Module):
314
next_tokens = self.sampler(logits, sampling_metadata)
315
return next_tokens
316
318
- def load_weights(self,
319
- model_name_or_path: str,
320
- cache_dir: Optional[str] = None,
321
- load_format: str = "auto",
322
- revision: Optional[str] = None):
317
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
318
stacked_params_mapping = [
319
# (param_name, shard_name, shard_id)
320
("qkv_proj", "q_proj", "q"),
@@ -327,8 +322,7 @@ class OPTForCausalLM(nn.Module):
322
("qkv_proj", "v_proj", "v"),
323
]
324
params_dict = dict(self.named_parameters(remove_duplicate=False))
330
- for name, loaded_weight in hf_model_weights_iterator(
331
- model_name_or_path, cache_dir, load_format, revision):
325
+ for name, loaded_weight in weights:
326
if "lm_head.weight" in name:
327
continue
328
if name.startswith("decoder."):
vllm/model_executor/models/orion.py
+4
-10
@@ -4,7 +4,7 @@
4
# Copyright (c) OrionStar Inc.
5
# LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE
6
"""Inference-only Orion-14B model compatible with HuggingFace weights."""
7
-from typing import Any, Dict, List, Optional, Tuple
7
+from typing import Any, Dict, Iterable, List, Optional, Tuple
8
9
import torch
10
from torch import nn
@@ -22,9 +22,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22
from vllm.model_executor.layers.sampler import Sampler
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
ParallelLMHead, VocabParallelEmbedding)
25
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
-from vllm.model_executor.weight_utils import (default_weight_loader,
27
- hf_model_weights_iterator)
27
from vllm.sequence import SamplerOutput
28
29
@@ -280,11 +279,7 @@ class OrionForCausalLM(nn.Module):
279
next_tokens = self.sampler(logits, sampling_metadata)
280
return next_tokens
281
283
- def load_weights(self,
284
- model_name_or_path: str,
285
- cache_dir: Optional[str] = None,
286
- load_format: str = "auto",
287
- revision: Optional[str] = None):
282
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
283
stacked_params_mapping = [
284
# (param_name, shard_name, shard_id)
285
("qkv_proj", "q_proj", "q"),
@@ -294,8 +289,7 @@ class OrionForCausalLM(nn.Module):
289
("gate_up_proj", "up_proj", 1),
290
]
291
params_dict = dict(self.named_parameters())
297
- for name, loaded_weight in hf_model_weights_iterator(
298
- model_name_or_path, cache_dir, load_format, revision):
292
+ for name, loaded_weight in weights:
293
if "rotary_emb.inv_freq" in name:
294
continue
295
if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/phi.py
+4
-10
@@ -35,7 +35,7 @@
35
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37
"""Inference-only Phi-1.5 model compatible with HuggingFace weights."""
38
-from typing import List, Optional
38
+from typing import Iterable, List, Optional, Tuple
39
40
import torch
41
from torch import nn
@@ -53,9 +53,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
53
from vllm.model_executor.layers.sampler import Sampler
54
from vllm.model_executor.layers.vocab_parallel_embedding import (
55
ParallelLMHead, VocabParallelEmbedding)
56
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
57
from vllm.model_executor.sampling_metadata import SamplingMetadata
57
-from vllm.model_executor.weight_utils import (default_weight_loader,
58
- hf_model_weights_iterator)
58
from vllm.sequence import SamplerOutput
59
60
@@ -265,11 +264,7 @@ class PhiForCausalLM(nn.Module):
264
next_tokens = self.sampler(logits, sampling_metadata)
265
return next_tokens
266
268
- def load_weights(self,
269
- model_name_or_path: str,
270
- cache_dir: Optional[str] = None,
271
- load_format: str = "auto",
272
- revision: Optional[str] = None):
267
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
268
stacked_params_mapping = [
269
# (param_name, shard_name, shard_id)
270
("qkv_proj", "q_proj", "q"),
@@ -278,8 +273,7 @@ class PhiForCausalLM(nn.Module):
273
]
274
params_dict = dict(self.named_parameters())
275
281
- for name, loaded_weight in hf_model_weights_iterator(
282
- model_name_or_path, cache_dir, load_format, revision):
276
+ for name, loaded_weight in weights:
277
if "rotary_emb.inv_freq" in name:
278
continue
279
vllm/model_executor/models/qwen.py
+4
-10
@@ -4,7 +4,7 @@
4
# Copyright (c) Alibaba Cloud.
5
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6
"""Inference-only QWen model compatible with HuggingFace weights."""
7
-from typing import Any, Dict, List, Optional, Tuple
7
+from typing import Any, Dict, Iterable, List, Optional, Tuple
8
9
import torch
10
from torch import nn
@@ -23,9 +23,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
23
from vllm.model_executor.layers.sampler import Sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
ParallelLMHead, VocabParallelEmbedding)
26
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
-from vllm.model_executor.weight_utils import (default_weight_loader,
28
- hf_model_weights_iterator)
28
from vllm.sequence import SamplerOutput
29
30
@@ -253,19 +252,14 @@ class QWenLMHeadModel(nn.Module):
252
next_tokens = self.sampler(logits, sampling_metadata)
253
return next_tokens
254
256
- def load_weights(self,
257
- model_name_or_path: str,
258
- cache_dir: Optional[str] = None,
259
- load_format: str = "auto",
260
- revision: Optional[str] = None):
255
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
256
stacked_params_mapping = [
257
# (param_name, shard_name, shard_id)
258
("gate_up_proj", "w2", 0),
259
("gate_up_proj", "w1", 1),
260
]
261
params_dict = dict(self.named_parameters())
267
- for name, loaded_weight in hf_model_weights_iterator(
268
- model_name_or_path, cache_dir, load_format, revision):
262
+ for name, loaded_weight in weights:
263
if "rotary_emb.inv_freq" in name:
264
continue
265
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/qwen2.py
+4
-10
@@ -22,7 +22,7 @@
22
# See the License for the specific language governing permissions and
23
# limitations under the License.
24
"""Inference-only Qwen2 model compatible with HuggingFace weights."""
25
-from typing import List, Optional, Tuple
25
+from typing import Iterable, List, Optional, Tuple
26
27
import torch
28
from torch import nn
@@ -42,9 +42,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
ParallelLMHead, VocabParallelEmbedding)
45
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.model_executor.weight_utils import (default_weight_loader,
47
- hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
48
49
@@ -331,11 +330,7 @@ class Qwen2ForCausalLM(nn.Module):
330
next_tokens = self.sampler(logits, sampling_metadata)
331
return next_tokens
332
334
- def load_weights(self,
335
- model_name_or_path: str,
336
- cache_dir: Optional[str] = None,
337
- load_format: str = "auto",
338
- revision: Optional[str] = None):
333
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
334
stacked_params_mapping = [
335
# (param_name, shard_name, shard_id)
336
("qkv_proj", "q_proj", "q"),
@@ -345,8 +340,7 @@ class Qwen2ForCausalLM(nn.Module):
340
("gate_up_proj", "up_proj", 1),
341
]
342
params_dict = dict(self.named_parameters(remove_duplicate=False))
348
- for name, loaded_weight in hf_model_weights_iterator(
349
- model_name_or_path, cache_dir, load_format, revision):
343
+ for name, loaded_weight in weights:
344
if "rotary_emb.inv_freq" in name:
345
continue
346
if self.config.tie_word_embeddings and "lm_head.weight" in name:
vllm/model_executor/models/qwen2_moe.py
+6
-14
@@ -22,7 +22,7 @@
22
# See the License for the specific language governing permissions and
23
# limitations under the License.
24
"""Inference-only Qwen2MoE model compatible with HuggingFace weights."""
25
-from typing import Any, Dict, List, Optional
25
+from typing import Any, Dict, Iterable, List, Optional, Tuple
26
27
import torch
28
import torch.nn.functional as F
@@ -46,9 +46,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
46
from vllm.model_executor.layers.sampler import Sampler
47
from vllm.model_executor.layers.vocab_parallel_embedding import (
48
ParallelLMHead, VocabParallelEmbedding)
49
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
-from vllm.model_executor.weight_utils import (default_weight_loader,
51
- hf_model_weights_iterator)
51
from vllm.sequence import SamplerOutput
52
53
@@ -366,6 +365,8 @@ class Qwen2MoeModel(nn.Module):
365
366
class Qwen2MoeForCausalLM(nn.Module):
367
368
+ fall_back_to_pt_during_load = False
369
+
370
def __init__(
371
self,
372
config: PretrainedConfig,
@@ -404,11 +405,7 @@ class Qwen2MoeForCausalLM(nn.Module):
405
next_tokens = self.sampler(logits, sampling_metadata)
406
return next_tokens
407
407
- def load_weights(self,
408
- model_name_or_path: str,
409
- cache_dir: Optional[str] = None,
410
- load_format: str = "auto",
411
- revision: Optional[str] = None):
408
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
409
stacked_params_mapping = [
410
# (param_name, shard_name, shard_id)
411
("qkv_proj", "q_proj", "q"),
@@ -419,12 +416,7 @@ class Qwen2MoeForCausalLM(nn.Module):
416
]
417
418
params_dict = dict(self.named_parameters())
422
- for name, loaded_weight in hf_model_weights_iterator(
423
- model_name_or_path,
424
- cache_dir,
425
- load_format,
426
- revision,
427
- fall_back_to_pt=False):
419
+ for name, loaded_weight in weights:
420
if "rotary_emb.inv_freq" in name:
421
continue
422
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/stablelm.py
+4
-10
@@ -19,7 +19,7 @@
19
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
20
"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM)
21
model compatible with HuggingFace weights."""
22
-from typing import List, Optional, Tuple
22
+from typing import Iterable, List, Optional, Tuple
23
24
import torch
25
from torch import nn
@@ -37,9 +37,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
-from vllm.model_executor.weight_utils import (default_weight_loader,
42
- hf_model_weights_iterator)
42
from vllm.sequence import SamplerOutput
43
44
@@ -262,11 +261,7 @@ class StablelmForCausalLM(nn.Module):
261
next_tokens = self.sampler(logits, sampling_metadata)
262
return next_tokens
263
265
- def load_weights(self,
266
- model_name_or_path: str,
267
- cache_dir: Optional[str] = None,
268
- load_format: str = "auto",
269
- revision: Optional[str] = None):
264
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
265
stacked_params_mapping = [
266
# (param_name, shard_name, shard_id)
267
("qkv_proj", "q_proj", "q"),
@@ -276,8 +271,7 @@ class StablelmForCausalLM(nn.Module):
271
("gate_up_proj", "up_proj", 1),
272
]
273
params_dict = dict(self.named_parameters())
279
- for name, loaded_weight in hf_model_weights_iterator(
280
- model_name_or_path, cache_dir, load_format, revision):
274
+ for name, loaded_weight in weights:
275
if "rotary_emb.inv_freq" in name:
276
continue
277
if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/starcoder2.py
+4
-10
@@ -18,7 +18,7 @@
18
# See the License for the specific language governing permissions and
19
# limitations under the License.
20
""" PyTorch Starcoder2 model."""
21
-from typing import List, Optional
21
+from typing import Iterable, List, Optional, Tuple
22
23
import torch
24
from torch import nn
@@ -36,9 +36,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
39
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
-from vllm.model_executor.weight_utils import (default_weight_loader,
41
- hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
43
@@ -274,11 +273,7 @@ class Starcoder2ForCausalLM(nn.Module):
273
next_tokens = self.sampler(logits, sampling_metadata)
274
return next_tokens
275
277
- def load_weights(self,
278
- model_name_or_path: str,
279
- cache_dir: Optional[str] = None,
280
- load_format: str = "auto",
281
- revision: Optional[str] = None):
276
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
277
stacked_params_mapping = [
278
# (param_name, shard_name, shard_id)
279
("qkv_proj", "q_proj", "q"),
@@ -287,8 +282,7 @@ class Starcoder2ForCausalLM(nn.Module):
282
]
283
284
params_dict = dict(self.named_parameters(remove_duplicate=False))
290
- for name, loaded_weight in hf_model_weights_iterator(
291
- model_name_or_path, cache_dir, load_format, revision):
285
+ for name, loaded_weight in weights:
286
if "rotary_emb.inv_freq" in name:
287
continue
288
vllm/model_executor/models/xverse.py
+4
-10
@@ -20,7 +20,7 @@
20
# See the License for the specific language governing permissions and
21
# limitations under the License.
22
"""Inference-only Xverse model compatible with HuggingFace weights."""
23
-from typing import Any, Dict, List, Optional, Tuple
23
+from typing import Any, Dict, Iterable, List, Optional, Tuple
24
25
import torch
26
from torch import nn
@@ -40,9 +40,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
43
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
-from vllm.model_executor.weight_utils import (default_weight_loader,
45
- hf_model_weights_iterator)
45
from vllm.sequence import SamplerOutput
46
47
@@ -331,11 +330,7 @@ class XverseForCausalLM(nn.Module):
330
next_tokens = self.sampler(logits, sampling_metadata)
331
return next_tokens
332
334
- def load_weights(self,
335
- model_name_or_path: str,
336
- cache_dir: Optional[str] = None,
337
- load_format: str = "auto",
338
- revision: Optional[str] = None):
333
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
334
stacked_params_mapping = [
335
("qkv_proj", "q_proj", "q"),
336
("qkv_proj", "k_proj", "k"),
@@ -344,8 +339,7 @@ class XverseForCausalLM(nn.Module):
339
("gate_up_proj", "up_proj", 1),
340
]
341
params_dict = dict(self.named_parameters())
347
- for name, loaded_weight in hf_model_weights_iterator(
348
- model_name_or_path, cache_dir, load_format, revision):
342
+ for name, loaded_weight in weights:
343
if ("rotary_emb.inv_freq" in name
344
or "rotary_emb.cos_cached" in name
345
or "rotary_emb.sin_cached" in name):
vllm/transformers_utils/tokenizer.py
+20
-1
@@ -1,8 +1,10 @@
1
+import os
2
from typing import Optional, Union
3
4
from transformers import (AutoTokenizer, PreTrainedTokenizer,
5
PreTrainedTokenizerFast)
6
7
+from vllm.config import VLLM_USE_MODELSCOPE
8
from vllm.logger import init_logger
9
from vllm.lora.request import LoRARequest
10
from vllm.transformers_utils.tokenizers import BaichuanTokenizer
@@ -57,9 +59,26 @@ def get_tokenizer(
59
tokenizer_mode: str = "auto",
60
trust_remote_code: bool = False,
61
tokenizer_revision: Optional[str] = None,
62
+ download_dir: Optional[str] = None,
63
**kwargs,
64
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
62
- """Gets a tokenizer for the given model name via Huggingface."""
65
+ """Gets a tokenizer for the given model name via Huggingface/modelscope."""
66
+ if VLLM_USE_MODELSCOPE:
67
+ # download model from ModelScope hub,
68
+ # lazy import so that modelscope is not required for normal use.
69
+ # pylint: disable=C.
70
+ from modelscope.hub.snapshot_download import snapshot_download
71
+
72
+ # Only set the tokenizer here, model will be downloaded on the workers.
73
+ if not os.path.exists(tokenizer_name):
74
+ tokenizer_path = snapshot_download(
75
+ model_id=tokenizer_name,
76
+ cache_dir=download_dir,
77
+ revision=tokenizer_revision,
78
+ # Ignore weights - we only need the tokenizer.
79
+ ignore_file_pattern=["*.pt", "*.safetensors", "*.bin"])
80
+ tokenizer_name = tokenizer_path
81
+
82
if tokenizer_mode == "slow":
83
if kwargs.get("use_fast", False):
84
raise ValueError(
vllm/worker/cpu_model_runner.py
+8
-4
@@ -3,8 +3,8 @@ from typing import Dict, List, Optional, Tuple
3
import torch
4
5
from vllm.attention import AttentionMetadata, get_attn_backend
6
-from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7
- SchedulerConfig)
6
+from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
7
+ ParallelConfig, SchedulerConfig)
8
from vllm.distributed import broadcast_tensor_dict
9
from vllm.logger import init_logger
10
from vllm.model_executor import SamplingMetadata
@@ -26,6 +26,7 @@ class CPUModelRunner:
26
parallel_config: ParallelConfig,
27
scheduler_config: SchedulerConfig,
28
device_config: DeviceConfig,
29
+ load_config: LoadConfig,
30
lora_config: Optional[LoRAConfig],
31
kv_cache_dtype: Optional[str] = "auto",
32
is_driver_worker: bool = False,
@@ -36,6 +37,7 @@ class CPUModelRunner:
37
self.parallel_config = parallel_config
38
self.scheduler_config = scheduler_config
39
self.lora_config = lora_config
40
+ self.load_config = load_config
41
self.is_driver_worker = is_driver_worker
42
43
# model_config can be None in tests/samplers/test_sampler.py.
@@ -55,8 +57,10 @@ class CPUModelRunner:
57
self.model_config.dtype if model_config is not None else None)
58
59
def load_model(self) -> None:
58
- self.model = get_model(self.model_config,
59
- self.device_config,
60
+ self.model = get_model(model_config=self.model_config,
61
+ load_config=self.load_config,
62
+ device_config=self.device_config,
63
+ vision_language_config=None,
64
lora_config=self.lora_config,
65
parallel_config=self.parallel_config,
66
scheduler_config=self.scheduler_config)
vllm/worker/cpu_worker.py
+5
-2
@@ -5,8 +5,8 @@ import torch
5
import torch.distributed
6
7
from vllm.attention import get_attn_backend
8
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
- ParallelConfig, SchedulerConfig)
8
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
+ ModelConfig, ParallelConfig, SchedulerConfig)
10
from vllm.distributed import (broadcast_tensor_dict,
11
ensure_model_parallel_initialized,
12
init_distributed_environment)
@@ -117,6 +117,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
117
scheduler_config: SchedulerConfig,
118
device_config: DeviceConfig,
119
cache_config: CacheConfig,
120
+ load_config: LoadConfig,
121
local_rank: int,
122
rank: int,
123
distributed_init_method: str,
@@ -129,6 +130,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
130
self.scheduler_config = scheduler_config
131
self.device_config = device_config
132
self.cache_config = cache_config
133
+ self.load_config = load_config
134
self.local_rank = local_rank
135
self.rank = rank
136
self.distributed_init_method = distributed_init_method
@@ -141,6 +143,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
143
parallel_config,
144
scheduler_config,
145
device_config,
146
+ load_config=self.load_config,
147
lora_config=self.lora_config,
148
kv_cache_dtype=kv_cache_dtype,
149
is_driver_worker=is_driver_worker)
vllm/worker/model_runner.py
+7
-8
@@ -9,9 +9,8 @@ import torch.nn as nn
9
10
from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11
get_attn_backend)
12
-from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
13
- SchedulerConfig, TensorizerConfig,
14
- VisionLanguageConfig)
12
+from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
13
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
14
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
15
from vllm.distributed.device_communicators import (custom_all_reduce,
16
pynccl_utils)
@@ -108,17 +107,17 @@ class ModelRunner:
107
parallel_config: ParallelConfig,
108
scheduler_config: SchedulerConfig,
109
device_config: DeviceConfig,
110
+ load_config: LoadConfig,
111
lora_config: Optional[LoRAConfig],
112
kv_cache_dtype: Optional[str] = "auto",
113
is_driver_worker: bool = False,
114
vision_language_config: Optional[VisionLanguageConfig] = None,
115
- tensorizer_config: Optional[TensorizerConfig] = None,
115
):
116
self.model_config = model_config
117
self.parallel_config = parallel_config
118
self.scheduler_config = scheduler_config
119
self.lora_config = lora_config
121
- self.tensorizer_config = tensorizer_config
120
+ self.load_config = load_config
121
self.is_driver_worker = is_driver_worker
122
123
# model_config can be None in tests/samplers/test_sampler.py.
@@ -156,13 +155,13 @@ class ModelRunner:
155
def load_model(self) -> None:
156
with CudaMemoryProfiler() as m:
157
self.model = get_model(
159
- self.model_config,
160
- self.device_config,
158
+ model_config=self.model_config,
159
+ device_config=self.device_config,
160
+ load_config=self.load_config,
161
lora_config=self.lora_config,
162
vision_language_config=self.vision_language_config,
163
parallel_config=self.parallel_config,
164
scheduler_config=self.scheduler_config,
165
- tensorizer_config=self.tensorizer_config,
165
)
166
167
self.model_memory_usage = m.consumed_memory
vllm/worker/neuron_model_runner.py
+1
-1
@@ -6,7 +6,7 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
6
SchedulerConfig)
7
from vllm.logger import init_logger
8
from vllm.model_executor import SamplingMetadata
9
-from vllm.model_executor.neuron_model_loader import get_neuron_model
9
+from vllm.model_executor.model_loader.neuron import get_neuron_model
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
12
from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
vllm/worker/worker.py
+5
-5
@@ -6,8 +6,8 @@ from typing import Dict, List, Optional, Set, Tuple
6
import torch
7
import torch.distributed
8
9
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
- ParallelConfig, SchedulerConfig, TensorizerConfig,
9
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
+ ModelConfig, ParallelConfig, SchedulerConfig,
11
VisionLanguageConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
@@ -38,12 +38,12 @@ class Worker(WorkerBase):
38
scheduler_config: SchedulerConfig,
39
device_config: DeviceConfig,
40
cache_config: CacheConfig,
41
+ load_config: LoadConfig,
42
local_rank: int,
43
rank: int,
44
distributed_init_method: str,
45
lora_config: Optional[LoRAConfig] = None,
46
vision_language_config: Optional[VisionLanguageConfig] = None,
46
- tensorizer_config: Optional[TensorizerConfig] = None,
47
is_driver_worker: bool = False,
48
) -> None:
49
self.model_config = model_config
@@ -55,7 +55,7 @@ class Worker(WorkerBase):
55
self.rank = rank
56
self.distributed_init_method = distributed_init_method
57
self.lora_config = lora_config
58
- self.tensorizer_config = tensorizer_config
58
+ self.load_config = load_config
59
self.is_driver_worker = is_driver_worker
60
if self.is_driver_worker:
61
assert self.rank == 0, "The driver worker must have rank 0."
@@ -70,11 +70,11 @@ class Worker(WorkerBase):
70
parallel_config,
71
scheduler_config,
72
device_config,
73
+ load_config=load_config,
74
lora_config=self.lora_config,
75
kv_cache_dtype=self.cache_config.cache_dtype,
76
is_driver_worker=is_driver_worker,
77
vision_language_config=vision_language_config,
77
- tensorizer_config=tensorizer_config,
78
)
79
# Uninitialized cache engine. Will be initialized by
80
# initialize_cache.