A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] PP support for embedding models and update docs (#9090)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Cyrus Leung committed
Oct 6, 2024 at 16:35 UTC
b22b79847153ae10710523cdb4a5fb98ac864cf4
12 files changed
+610
-449
docs/source/models/supported_models.rst
+56
-4
@@ -7,10 +7,12 @@ vLLM supports a variety of generative Transformer models in `HuggingFace Transfo
7
The following is the list of model architectures that are currently supported by vLLM.
8
Alongside each architecture, we include some popular models that use it.
9
10
-----
10
+Text-only Language Models
11
+^^^^^^^^^^^^^^^^^^^^^^^^^
12
+
13
+Text Generation
14
+---------------
15
12
-Decoder-only Language Models
13
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
16
.. list-table::
17
:widths: 25 25 50 5 5
18
:header-rows: 1
@@ -40,6 +42,11 @@ Decoder-only Language Models
42
- :code:`bigscience/bloom`, :code:`bigscience/bloomz`, etc.
43
-
44
- ✅︎
45
+ * - :code:`BartForConditionalGeneration`
46
+ - BART
47
+ - :code:`facebook/bart-base`, :code:`facebook/bart-large-cnn`, etc.
48
+ -
49
+ -
50
* - :code:`ChatGLMModel`
51
- ChatGLM
52
- :code:`THUDM/chatglm2-6b`, :code:`THUDM/chatglm3-6b`, etc.
@@ -259,11 +266,55 @@ Decoder-only Language Models
266
.. note::
267
Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
268
262
-.. _supported_vlms:
269
+Text Embedding
270
+--------------
271
+
272
+.. list-table::
273
+ :widths: 25 25 50 5 5
274
+ :header-rows: 1
275
+
276
+ * - Architecture
277
+ - Models
278
+ - Example HuggingFace Models
279
+ - :ref:`LoRA <lora>`
280
+ - :ref:`PP <distributed_serving>`
281
+ * - :code:`Gemma2Model`
282
+ - Gemma2-based
283
+ - :code:`BAAI/bge-multilingual-gemma2`, etc.
284
+ -
285
+ - ✅︎
286
+ * - :code:`MistralModel`
287
+ - Mistral-based
288
+ - :code:`intfloat/e5-mistral-7b-instruct`, etc.
289
+ -
290
+ - ✅︎
291
+
292
+Reward Modeling
293
+---------------
294
+
295
+.. list-table::
296
+ :widths: 25 25 50 5 5
297
+ :header-rows: 1
298
+
299
+ * - Architecture
300
+ - Models
301
+ - Example HuggingFace Models
302
+ - :ref:`LoRA <lora>`
303
+ - :ref:`PP <distributed_serving>`
304
+ * - :code:`Qwen2ForRewardModel`
305
+ - Qwen2-based
306
+ - :code:`Qwen/Qwen2.5-Math-RM-72B`, etc.
307
+ -
308
+ - ✅︎
309
+
310
+.. note::
311
+ As an interim measure, these models are supported via Embeddings API. See `this RFC <https://github.com/vllm-project/vllm/issues/8967>`_ for upcoming changes.
312
313
Multimodal Language Models
314
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
315
316
+.. _supported_vlms:
317
+
318
.. list-table::
319
:widths: 25 25 25 25 5 5
320
:header-rows: 1
@@ -378,6 +429,7 @@ Multimodal Language Models
429
For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
430
For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
431
432
+----
433
434
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
435
Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
docs/source/models/vlm.rst
+3
-4
@@ -6,10 +6,9 @@ Using VLMs
6
vLLM provides experimental support for Vision Language Models (VLMs). See the :ref:`list of supported VLMs here <supported_vlms>`.
7
This document shows you how to run and serve these models using vLLM.
8
9
-.. important::
10
- We are actively iterating on VLM support. Expect breaking changes to VLM usage and development in upcoming releases without prior deprecation.
11
-
12
- We are continuously improving user & developer experience for VLMs. Please `open an issue on GitHub <https://github.com/vllm-project/vllm/issues/new/choose>`_ if you have any feedback or feature requests.
9
+.. note::
10
+ We are actively iterating on VLM support. See `this RFC <https://github.com/vllm-project/vllm/issues/4194>`_ for upcoming changes,
11
+ and `open an issue on GitHub <https://github.com/vllm-project/vllm/issues/new/choose>`_ if you have any feedback or feature requests.
12
13
Offline Inference
14
-----------------
tests/distributed/test_pipeline_parallel.py
+114
-32
@@ -7,7 +7,7 @@ WARNING: This test runs in both single-node (4 GPUs) and multi-node
7
"""
8
import os
9
from dataclasses import dataclass
10
-from typing import List, NamedTuple, Optional
10
+from typing import List, Literal, NamedTuple, Optional
11
12
import pytest
13
@@ -97,6 +97,9 @@ class PPTestSettings:
97
self.trust_remote_code, self.tokenizer_mode)
98
99
100
+# NOTE: You can adjust tp_base and/or pp_base locally to fit the model in GPU
101
+# The values displayed here are only a rough indicator of the size of the model
102
+
103
# yapf: disable
104
GENERATION_MODEL_SETTINGS = {
105
# [DETAILED TESTS]
@@ -104,15 +107,13 @@ GENERATION_MODEL_SETTINGS = {
107
# [FAST TESTS]
108
# Uses Llama
109
# "BAAI/AquilaChat-7B": PPTestSettings.fast(),
107
- # TODO: Test on larger GPU
108
- # "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
110
+ "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(tp_base=8, trust_remote_code=True), # noqa: E501
111
"baichuan-inc/Baichuan-7B": PPTestSettings.fast(trust_remote_code=True),
112
"baichuan-inc/Baichuan2-13B-Chat": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
113
"bigscience/bloomz-1b1": PPTestSettings.fast(),
114
"THUDM/chatglm3-6b": PPTestSettings.fast(trust_remote_code=True),
115
"CohereForAI/c4ai-command-r-v01": PPTestSettings.fast(tp_base=2, trust_remote_code=True), # noqa: E501
114
- # TODO: Test on larger GPU
115
- # "databricks/dbrx-instruct": PPTestSettings.fast(),
116
+ "databricks/dbrx-instruct": PPTestSettings.fast(tp_base=8),
117
"Deci/DeciLM-7B-instruct": PPTestSettings.fast(trust_remote_code=True),
118
"deepseek-ai/deepseek-llm-7b-chat": PPTestSettings.fast(),
119
"deepseek-ai/DeepSeek-V2-Lite-Chat": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
@@ -161,8 +162,9 @@ GENERATION_MODEL_SETTINGS = {
162
163
EMBEDDING_MODEL_SETTINGS = { # type: ignore[var-annotated]
164
# [FAST TESTS]
164
- # Uses Llama
165
- # "intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(),
165
+ "intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(),
166
+ "BAAI/bge-multilingual-gemma2": PPTestSettings.fast(),
167
+ "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(tp_base=4, trust_remote_code=True), # noqa: E501
168
}
169
170
MULTIMODAL_MODEL_SETTINGS = {
@@ -192,40 +194,35 @@ CONDITIONAL_GENERATION_MODEL_SETTINGS = { # type: ignore[var-annotated]
194
}
195
# yapf: enable
196
195
-MODEL_SETTINGS = {
196
- **GENERATION_MODEL_SETTINGS,
197
- **EMBEDDING_MODEL_SETTINGS,
198
- **MULTIMODAL_MODEL_SETTINGS,
199
-}
200
-
201
-# You can update this on your local machine to run specific tests
197
+# NOTE: You can update this on your local machine to run specific tests
198
TEST_MODELS = [
199
+ # [LANGUAGE GENERATION]
200
"meta-llama/Meta-Llama-3-8B",
204
- "facebook/chameleon-7b",
201
+ "ibm/PowerLM-3b",
202
+ # [LANGUAGE EMBEDDING]
203
+ "intfloat/e5-mistral-7b-instruct",
204
+ "BAAI/bge-multilingual-gemma2",
205
+ # [MULTIMODAL GENERATION]
206
"OpenGVLab/InternVL2-1B",
207
"microsoft/Phi-3-vision-128k-instruct",
207
- "mistralai/Pixtral-12B-2409",
208
"fixie-ai/ultravox-v0_3",
209
]
210
211
212
-@pytest.mark.parametrize(
213
- ("model_name", "parallel_setup", "distributed_backend",
214
- "trust_remote_code", "tokenizer_mode"),
215
- [
216
- params for model_name, settings in MODEL_SETTINGS.items()
217
- for params in settings.iter_params(model_name)
218
- if model_name in TEST_MODELS
219
- ],
220
-)
221
-@fork_new_process_for_each_test
222
-def test_compare_tp(model_name: str, parallel_setup: ParallelSetup,
223
- distributed_backend: str, trust_remote_code: bool,
224
- tokenizer_mode: Optional[str], num_gpus_available):
212
+def _compare_tp(
213
+ model_name: str,
214
+ parallel_setup: ParallelSetup,
215
+ distributed_backend: str,
216
+ trust_remote_code: bool,
217
+ tokenizer_mode: Optional[str],
218
+ num_gpus_available: int,
219
+ *,
220
+ method: Literal["generate", "encode"] = "encode",
221
+):
222
tp_size, pp_size, eager_mode, chunked_prefill = parallel_setup
223
227
- if num_gpus_available < tp_size:
228
- pytest.skip(f"Need at least {tp_size} GPUs to run the test")
224
+ if num_gpus_available < tp_size * pp_size:
225
+ pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
226
if VLLM_MULTI_NODE and distributed_backend == "mp":
227
pytest.skip("Skipping multi-node pipeline parallel test for "
228
"multiprocessing distributed backend")
@@ -286,10 +283,95 @@ def test_compare_tp(model_name: str, parallel_setup: ParallelSetup,
283
]
284
285
try:
289
- compare_two_settings(model_name, pp_args, tp_args, pp_env)
286
+ compare_two_settings(model_name,
287
+ pp_args,
288
+ tp_args,
289
+ pp_env,
290
+ method=method)
291
except Exception:
292
if pp_env is None:
293
raise
294
else:
295
# Ray ADAG tests are flaky, so we don't want to fail the test
296
logger.exception("Ray ADAG tests failed")
297
+
298
+
299
+@pytest.mark.parametrize(
300
+ ("model_name", "parallel_setup", "distributed_backend",
301
+ "trust_remote_code", "tokenizer_mode"),
302
+ [
303
+ params for model_name, settings in GENERATION_MODEL_SETTINGS.items()
304
+ for params in settings.iter_params(model_name)
305
+ if model_name in TEST_MODELS
306
+ ],
307
+)
308
+@fork_new_process_for_each_test
309
+def test_tp_language_generation(
310
+ model_name: str,
311
+ parallel_setup: ParallelSetup,
312
+ distributed_backend: str,
313
+ trust_remote_code: bool,
314
+ tokenizer_mode: Optional[str],
315
+ num_gpus_available,
316
+):
317
+ _compare_tp(model_name,
318
+ parallel_setup,
319
+ distributed_backend,
320
+ trust_remote_code,
321
+ tokenizer_mode,
322
+ num_gpus_available,
323
+ method="generate")
324
+
325
+
326
+@pytest.mark.parametrize(
327
+ ("model_name", "parallel_setup", "distributed_backend",
328
+ "trust_remote_code", "tokenizer_mode"),
329
+ [
330
+ params for model_name, settings in EMBEDDING_MODEL_SETTINGS.items()
331
+ for params in settings.iter_params(model_name)
332
+ if model_name in TEST_MODELS
333
+ ],
334
+)
335
+@fork_new_process_for_each_test
336
+def test_tp_language_embedding(
337
+ model_name: str,
338
+ parallel_setup: ParallelSetup,
339
+ distributed_backend: str,
340
+ trust_remote_code: bool,
341
+ tokenizer_mode: Optional[str],
342
+ num_gpus_available,
343
+):
344
+ _compare_tp(model_name,
345
+ parallel_setup,
346
+ distributed_backend,
347
+ trust_remote_code,
348
+ tokenizer_mode,
349
+ num_gpus_available,
350
+ method="encode")
351
+
352
+
353
+@pytest.mark.parametrize(
354
+ ("model_name", "parallel_setup", "distributed_backend",
355
+ "trust_remote_code", "tokenizer_mode"),
356
+ [
357
+ params for model_name, settings in MULTIMODAL_MODEL_SETTINGS.items()
358
+ for params in settings.iter_params(model_name)
359
+ if model_name in TEST_MODELS
360
+ ],
361
+)
362
+@fork_new_process_for_each_test
363
+def test_tp_multimodal_generation(
364
+ model_name: str,
365
+ parallel_setup: ParallelSetup,
366
+ distributed_backend: str,
367
+ trust_remote_code: bool,
368
+ tokenizer_mode: Optional[str],
369
+ num_gpus_available,
370
+):
371
+ _compare_tp(model_name,
372
+ parallel_setup,
373
+ distributed_backend,
374
+ trust_remote_code,
375
+ tokenizer_mode,
376
+ num_gpus_available,
377
+ method="generate")
tests/utils.py
+137
-92
@@ -8,13 +8,13 @@ import time
8
import warnings
9
from contextlib import contextmanager
10
from pathlib import Path
11
-from typing import Any, Callable, Dict, List, Optional, Union
11
+from typing import Any, Callable, Dict, List, Literal, Optional, Union
12
13
import openai
14
import pytest
15
import requests
16
from openai.types.completion import Completion
17
-from typing_extensions import ParamSpec
17
+from typing_extensions import ParamSpec, assert_never
18
19
from tests.models.utils import TextTextLogprobs
20
from vllm.distributed import (ensure_model_parallel_initialized,
@@ -163,11 +163,140 @@ class RemoteOpenAIServer:
163
)
164
165
166
+def _test_completion(
167
+ client: openai.OpenAI,
168
+ model: str,
169
+ prompt: str,
170
+ token_ids: List[int],
171
+):
172
+ results = []
173
+
174
+ # test with text prompt
175
+ completion = client.completions.create(model=model,
176
+ prompt=prompt,
177
+ max_tokens=5,
178
+ temperature=0.0)
179
+
180
+ results.append({
181
+ "test": "single_completion",
182
+ "text": completion.choices[0].text,
183
+ "finish_reason": completion.choices[0].finish_reason,
184
+ "usage": completion.usage,
185
+ })
186
+
187
+ # test using token IDs
188
+ completion = client.completions.create(
189
+ model=model,
190
+ prompt=token_ids,
191
+ max_tokens=5,
192
+ temperature=0.0,
193
+ )
194
+
195
+ results.append({
196
+ "test": "token_ids",
197
+ "text": completion.choices[0].text,
198
+ "finish_reason": completion.choices[0].finish_reason,
199
+ "usage": completion.usage,
200
+ })
201
+
202
+ # test seeded random sampling
203
+ completion = client.completions.create(model=model,
204
+ prompt=prompt,
205
+ max_tokens=5,
206
+ seed=33,
207
+ temperature=1.0)
208
+
209
+ results.append({
210
+ "test": "seeded_sampling",
211
+ "text": completion.choices[0].text,
212
+ "finish_reason": completion.choices[0].finish_reason,
213
+ "usage": completion.usage,
214
+ })
215
+
216
+ # test seeded random sampling with multiple prompts
217
+ completion = client.completions.create(model=model,
218
+ prompt=[prompt, prompt],
219
+ max_tokens=5,
220
+ seed=33,
221
+ temperature=1.0)
222
+
223
+ results.append({
224
+ "test":
225
+ "seeded_sampling",
226
+ "text": [choice.text for choice in completion.choices],
227
+ "finish_reason":
228
+ [choice.finish_reason for choice in completion.choices],
229
+ "usage":
230
+ completion.usage,
231
+ })
232
+
233
+ # test simple list
234
+ batch = client.completions.create(
235
+ model=model,
236
+ prompt=[prompt, prompt],
237
+ max_tokens=5,
238
+ temperature=0.0,
239
+ )
240
+
241
+ results.append({
242
+ "test": "simple_list",
243
+ "text0": batch.choices[0].text,
244
+ "text1": batch.choices[1].text,
245
+ })
246
+
247
+ # test streaming
248
+ batch = client.completions.create(
249
+ model=model,
250
+ prompt=[prompt, prompt],
251
+ max_tokens=5,
252
+ temperature=0.0,
253
+ stream=True,
254
+ )
255
+
256
+ texts = [""] * 2
257
+ for chunk in batch:
258
+ assert len(chunk.choices) == 1
259
+ choice = chunk.choices[0]
260
+ texts[choice.index] += choice.text
261
+
262
+ results.append({
263
+ "test": "streaming",
264
+ "texts": texts,
265
+ })
266
+
267
+ return results
268
+
269
+
270
+def _test_embeddings(
271
+ client: openai.OpenAI,
272
+ model: str,
273
+ text: str,
274
+):
275
+ results = []
276
+
277
+ # test with text input
278
+ embeddings = client.embeddings.create(
279
+ model=model,
280
+ input=text,
281
+ encoding_format="float",
282
+ )
283
+
284
+ results.append({
285
+ "test": "single_embedding",
286
+ "embedding": embeddings.data[0].embedding,
287
+ "usage": embeddings.usage,
288
+ })
289
+
290
+ return results
291
+
292
+
293
def compare_two_settings(model: str,
294
arg1: List[str],
295
arg2: List[str],
296
env1: Optional[Dict[str, str]] = None,
297
env2: Optional[Dict[str, str]] = None,
298
+ *,
299
+ method: Literal["generate", "encode"] = "generate",
300
max_wait_seconds: Optional[float] = None) -> None:
301
"""
302
Launch API server with two different sets of arguments/environments
@@ -219,96 +348,12 @@ def compare_two_settings(model: str,
348
"root": served_model.root,
349
})
350
222
- # test with text prompt
223
- completion = client.completions.create(model=model,
224
- prompt=prompt,
225
- max_tokens=5,
226
- temperature=0.0)
227
-
228
- results.append({
229
- "test": "single_completion",
230
- "text": completion.choices[0].text,
231
- "finish_reason": completion.choices[0].finish_reason,
232
- "usage": completion.usage,
233
- })
234
-
235
- # test using token IDs
236
- completion = client.completions.create(
237
- model=model,
238
- prompt=token_ids,
239
- max_tokens=5,
240
- temperature=0.0,
241
- )
242
-
243
- results.append({
244
- "test": "token_ids",
245
- "text": completion.choices[0].text,
246
- "finish_reason": completion.choices[0].finish_reason,
247
- "usage": completion.usage,
248
- })
249
-
250
- # test seeded random sampling
251
- completion = client.completions.create(model=model,
252
- prompt=prompt,
253
- max_tokens=5,
254
- seed=33,
255
- temperature=1.0)
256
-
257
- results.append({
258
- "test": "seeded_sampling",
259
- "text": completion.choices[0].text,
260
- "finish_reason": completion.choices[0].finish_reason,
261
- "usage": completion.usage,
262
- })
263
-
264
- # test seeded random sampling with multiple prompts
265
- completion = client.completions.create(model=model,
266
- prompt=[prompt, prompt],
267
- max_tokens=5,
268
- seed=33,
269
- temperature=1.0)
270
-
271
- results.append({
272
- "test":
273
- "seeded_sampling",
274
- "text": [choice.text for choice in completion.choices],
275
- "finish_reason":
276
- [choice.finish_reason for choice in completion.choices],
277
- "usage":
278
- completion.usage,
279
- })
280
-
281
- # test simple list
282
- batch = client.completions.create(
283
- model=model,
284
- prompt=[prompt, prompt],
285
- max_tokens=5,
286
- temperature=0.0,
287
- )
288
-
289
- results.append({
290
- "test": "simple_list",
291
- "text0": batch.choices[0].text,
292
- "text1": batch.choices[1].text,
293
- })
294
-
295
- # test streaming
296
- batch = client.completions.create(
297
- model=model,
298
- prompt=[prompt, prompt],
299
- max_tokens=5,
300
- temperature=0.0,
301
- stream=True,
302
- )
303
- texts = [""] * 2
304
- for chunk in batch:
305
- assert len(chunk.choices) == 1
306
- choice = chunk.choices[0]
307
- texts[choice.index] += choice.text
308
- results.append({
309
- "test": "streaming",
310
- "texts": texts,
311
- })
351
+ if method == "generate":
352
+ results += _test_completion(client, model, prompt, token_ids)
353
+ elif method == "encode":
354
+ results += _test_embeddings(client, model, prompt)
355
+ else:
356
+ assert_never(method)
357
358
n = len(results) // 2
359
arg1_results = results[:n]
vllm/model_executor/models/gemma2.py
+57
-43
@@ -40,7 +40,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.sequence import IntermediateTensors
41
42
from .interfaces import SupportsLoRA, SupportsPP
43
-from .utils import (is_pp_missing_parameter,
43
+from .utils import (group_weights_with_prefix, is_pp_missing_parameter,
44
make_empty_intermediate_tensors_factory, make_layers)
45
46
logger = init_logger(__name__)
@@ -273,7 +273,7 @@ class Gemma2Model(nn.Module):
273
274
def forward(
275
self,
276
- input_ids: torch.Tensor,
276
+ input_ids: Optional[torch.Tensor],
277
positions: torch.Tensor,
278
kv_caches: List[torch.Tensor],
279
attn_metadata: AttentionMetadata,
@@ -308,6 +308,49 @@ class Gemma2Model(nn.Module):
308
hidden_states, _ = self.norm(hidden_states, residual)
309
return hidden_states
310
311
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
312
+ stacked_params_mapping = [
313
+ # (param_name, shard_name, shard_id)
314
+ ("qkv_proj", "q_proj", "q"),
315
+ ("qkv_proj", "k_proj", "k"),
316
+ ("qkv_proj", "v_proj", "v"),
317
+ ("gate_up_proj", "gate_proj", 0),
318
+ ("gate_up_proj", "up_proj", 1),
319
+ ]
320
+ params_dict = dict(self.named_parameters())
321
+ loaded_params: Set[str] = set()
322
+ for name, loaded_weight in weights:
323
+ for (param_name, shard_name, shard_id) in stacked_params_mapping:
324
+ if shard_name not in name:
325
+ continue
326
+ name = name.replace(shard_name, param_name)
327
+ # Skip loading extra bias for GPTQ models.
328
+ if name.endswith(".bias") and name not in params_dict:
329
+ continue
330
+ if is_pp_missing_parameter(name, self):
331
+ continue
332
+ param = params_dict[name]
333
+ weight_loader = param.weight_loader
334
+ weight_loader(param, loaded_weight, shard_id)
335
+ break
336
+ else:
337
+ # Skip loading extra bias for GPTQ models.
338
+ if name.endswith(".bias") and name not in params_dict:
339
+ continue
340
+ if is_pp_missing_parameter(name, self):
341
+ continue
342
+ param = params_dict[name]
343
+ weight_loader = getattr(param, "weight_loader",
344
+ default_weight_loader)
345
+ weight_loader(param, loaded_weight)
346
+ loaded_params.add(name)
347
+
348
+ unloaded_params = params_dict.keys() - loaded_params
349
+ if unloaded_params:
350
+ logger.warning(
351
+ "Some weights are not initialized from checkpoints: %s",
352
+ unloaded_params)
353
+
354
355
class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
356
packed_modules_mapping = {
@@ -391,48 +434,19 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
434
return next_tokens
435
436
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
394
- stacked_params_mapping = [
395
- # (param_name, shard_name, shard_id)
396
- ("qkv_proj", "q_proj", "q"),
397
- ("qkv_proj", "k_proj", "k"),
398
- ("qkv_proj", "v_proj", "v"),
399
- ("gate_up_proj", "gate_proj", 0),
400
- ("gate_up_proj", "up_proj", 1),
401
- ]
402
- params_dict = dict(self.named_parameters())
403
- loaded_params: Set[str] = set()
404
- for name, loaded_weight in weights:
405
- for (param_name, shard_name, shard_id) in stacked_params_mapping:
406
- if shard_name not in name:
407
- continue
408
- name = name.replace(shard_name, param_name)
409
- # Skip loading extra bias for GPTQ models.
410
- if name.endswith(".bias") and name not in params_dict:
411
- continue
412
- if is_pp_missing_parameter(name, self):
413
- continue
414
- param = params_dict[name]
415
- weight_loader = param.weight_loader
416
- weight_loader(param, loaded_weight, shard_id)
417
- break
418
- else:
419
- # lm_head is not used in vllm as it is tied with embed_token.
420
- # To prevent errors, skip loading lm_head.weight.
421
- if "lm_head.weight" in name:
422
- continue
423
- # Skip loading extra bias for GPTQ models.
424
- if name.endswith(".bias") and name not in params_dict:
425
- continue
426
- if is_pp_missing_parameter(name, self):
437
+ weights_group = group_weights_with_prefix(weights)
438
+
439
+ self.model.load_weights(weights_group["model"])
440
+
441
+ if not self.config.tie_word_embeddings:
442
+ # NOTE: For now self.lm_head is not defined because
443
+ # tie_word_embeddings is assumed to the False
444
+ lm_head_dict = dict(self.lm_head.named_parameters())
445
+ for name, loaded_weight in weights_group["lm_head"]:
446
+ if is_pp_missing_parameter(name, self.lm_head):
447
continue
428
- param = params_dict[name]
448
+
449
+ param = lm_head_dict[name]
450
weight_loader = getattr(param, "weight_loader",
451
default_weight_loader)
452
weight_loader(param, loaded_weight)
432
- loaded_params.add(name)
433
-
434
- unloaded_params = params_dict.keys() - loaded_params
435
- if unloaded_params:
436
- logger.warning(
437
- "Some weights are not initialized from checkpoints: %s",
438
- unloaded_params)
vllm/model_executor/models/gemma2_embedding.py
+12
-37
@@ -1,17 +1,18 @@
1
-from typing import Iterable, List, Optional, Tuple
1
+from typing import Iterable, List, Optional, Tuple, Union
2
3
import torch
4
from torch import nn
5
6
from vllm.attention import AttentionMetadata
7
from vllm.model_executor.layers.pooler import Pooler, PoolingType
8
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
9
-from vllm.model_executor.models.gemma2 import Gemma2Model
8
from vllm.model_executor.pooling_metadata import PoolingMetadata
9
from vllm.sequence import IntermediateTensors, PoolerOutput
10
11
+from .gemma2 import Gemma2Model
12
+from .interfaces import SupportsPP
13
14
-class Gemma2EmbeddingModel(nn.Module):
14
+
15
+class Gemma2EmbeddingModel(nn.Module, SupportsPP):
16
"""A model that uses Gemma2 with additional embedding functionalities.
17
18
This class encapsulates the Gemma2Model and provides an interface for
@@ -30,6 +31,9 @@ class Gemma2EmbeddingModel(nn.Module):
31
self.model = Gemma2Model(**kwargs)
32
self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
33
34
+ self.make_empty_intermediate_tensors = (
35
+ self.model.make_empty_intermediate_tensors)
36
+
37
def forward(
38
self,
39
input_ids: Optional[torch.Tensor],
@@ -38,10 +42,9 @@ class Gemma2EmbeddingModel(nn.Module):
42
attn_metadata: AttentionMetadata,
43
intermediate_tensors: Optional[IntermediateTensors] = None,
44
inputs_embeds: Optional[torch.Tensor] = None,
41
- ) -> torch.Tensor:
42
- return self.model.forward(input_ids, positions, kv_caches,
43
- attn_metadata, intermediate_tensors,
44
- inputs_embeds)
45
+ ) -> Union[torch.Tensor, IntermediateTensors]:
46
+ return self.model(input_ids, positions, kv_caches, attn_metadata,
47
+ intermediate_tensors, inputs_embeds)
48
49
def pooler(
50
self,
@@ -51,32 +54,4 @@ class Gemma2EmbeddingModel(nn.Module):
54
return self._pooler(hidden_states, pooling_metadata)
55
56
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
54
- stacked_params_mapping = [
55
- # (param_name, shard_name, shard_id)
56
- ("qkv_proj", "q_proj", "q"),
57
- ("qkv_proj", "k_proj", "k"),
58
- ("qkv_proj", "v_proj", "v"),
59
- ("gate_up_proj", "gate_proj", 0),
60
- ("gate_up_proj", "up_proj", 1),
61
- ]
62
- params_dict = dict(self.model.named_parameters())
63
- for name, loaded_weight in weights:
64
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
65
- if weight_name not in name:
66
- continue
67
- name = name.replace(weight_name, param_name)
68
- # Skip loading extra bias for GPTQ models.
69
- if name.endswith(".bias") and name not in params_dict:
70
- continue
71
- param = params_dict[name]
72
- weight_loader = param.weight_loader
73
- weight_loader(param, loaded_weight, shard_id)
74
- break
75
- else:
76
- # Skip loading extra bias for GPTQ models.
77
- if name.endswith(".bias") and name not in params_dict:
78
- continue
79
- param = params_dict[name]
80
- weight_loader = getattr(param, "weight_loader",
81
- default_weight_loader)
82
- weight_loader(param, loaded_weight)
57
+ self.model.load_weights(weights)
vllm/model_executor/models/llama.py
+103
-82
@@ -51,7 +51,8 @@ from vllm.sequence import IntermediateTensors
51
from vllm.utils import is_hip
52
53
from .interfaces import SupportsLoRA, SupportsPP
54
-from .utils import (PPMissingLayer, is_pp_missing_parameter,
54
+from .utils import (PPMissingLayer, group_weights_with_prefix,
55
+ is_pp_missing_parameter,
56
make_empty_intermediate_tensors_factory, make_layers)
57
58
@@ -347,6 +348,90 @@ class LlamaModel(nn.Module):
348
hidden_states, _ = self.norm(hidden_states, residual)
349
return hidden_states
350
351
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
352
+ stacked_params_mapping = [
353
+ # (param_name, shard_name, shard_id)
354
+ (".qkv_proj", ".q_proj", "q"),
355
+ (".qkv_proj", ".k_proj", "k"),
356
+ (".qkv_proj", ".v_proj", "v"),
357
+ (".gate_up_proj", ".gate_proj", 0),
358
+ (".gate_up_proj", ".up_proj", 1),
359
+ ]
360
+ params_dict = dict(self.named_parameters())
361
+ for name, loaded_weight in weights:
362
+ if "rotary_emb.inv_freq" in name:
363
+ continue
364
+ if ("rotary_emb.cos_cached" in name
365
+ or "rotary_emb.sin_cached" in name):
366
+ # Models trained using ColossalAI may include these tensors in
367
+ # the checkpoint. Skip them.
368
+ continue
369
+ if scale_name := get_compressed_tensors_cache_scale(name):
370
+ # Loading kv cache scales for compressed-tensors quantization
371
+ param = params_dict[scale_name]
372
+ weight_loader = getattr(param, "weight_loader",
373
+ default_weight_loader)
374
+ loaded_weight = loaded_weight[0]
375
+ weight_loader(param, loaded_weight)
376
+ continue
377
+ for param_name, weight_name, shard_id in stacked_params_mapping:
378
+ if weight_name not in name:
379
+ continue
380
+ name = name.replace(weight_name, param_name)
381
+ # Skip loading extra bias for GPTQ models.
382
+ if name.endswith(".bias") and name not in params_dict:
383
+ continue
384
+
385
+ if is_pp_missing_parameter(name, self):
386
+ continue
387
+
388
+ param = params_dict[name]
389
+ weight_loader = param.weight_loader
390
+ weight_loader(param, loaded_weight, shard_id)
391
+
392
+ break
393
+ else:
394
+ # Skip loading extra bias for GPTQ models.
395
+ if name.endswith(".bias") and name not in params_dict:
396
+ continue
397
+ # Remapping the name of FP8 kv-scale.
398
+ name = maybe_remap_kv_scale_name(name, params_dict)
399
+ if name is None:
400
+ continue
401
+
402
+ if is_pp_missing_parameter(name, self):
403
+ continue
404
+
405
+ param = params_dict[name]
406
+ weight_loader = getattr(param, "weight_loader",
407
+ default_weight_loader)
408
+ weight_loader(param, loaded_weight)
409
+
410
+ # If this function is called, it should always initialize KV cache scale
411
+ # factors (or else raise an exception). Thus, handled exceptions should
412
+ # make sure to leave KV cache scale factors in a known good (dummy) state
413
+ def load_kv_cache_scales(self, quantization_param_path: str) -> None:
414
+ tp_size = get_tensor_model_parallel_world_size()
415
+ tp_rank = get_tensor_model_parallel_rank()
416
+ for layer_idx, scaling_factor in kv_cache_scales_loader(
417
+ quantization_param_path, tp_rank, tp_size,
418
+ self.config.num_hidden_layers,
419
+ self.config.__class__.model_type):
420
+ if not isinstance(self.layers[layer_idx], nn.Identity):
421
+ layer_self_attn = self.layers[layer_idx].self_attn
422
+
423
+ if is_hip():
424
+ # The scaling factor convention we are assuming is
425
+ # quantized_value * scaling_factor ~= true_value
426
+ # which is consistent with the practice of setting
427
+ # scaling_factor = tensor_amax / FPtype_max
428
+ scaling_factor *= 2
429
+ if hasattr(layer_self_attn, "kv_scale"):
430
+ layer_self_attn.attn._kv_scale = scaling_factor
431
+ else:
432
+ raise RuntimeError("Self attention has no KV cache scaling "
433
+ "factor attribute!")
434
+
435
436
class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
437
packed_modules_mapping = {
@@ -372,6 +457,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
457
"gate_proj": ("gate_up_proj", 0),
458
"up_proj": ("gate_up_proj", 1),
459
}
460
+
461
# Mistral/Llama models can also be loaded with --load-format mistral
462
# from consolidated.safetensors checkpoints
463
mistral_mapping = {
@@ -465,103 +551,38 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
551
return next_tokens
552
553
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
468
- stacked_params_mapping = [
469
- # (param_name, shard_name, shard_id)
470
- (".qkv_proj", ".q_proj", "q"),
471
- (".qkv_proj", ".k_proj", "k"),
472
- (".qkv_proj", ".v_proj", "v"),
473
- (".gate_up_proj", ".gate_proj", 0),
474
- (".gate_up_proj", ".up_proj", 1),
554
+ weights = [
555
+ self.maybe_remap_mistral(name, loaded_weight)
556
+ for name, loaded_weight in weights
557
]
476
- params_dict = dict(self.named_parameters())
477
- for name, loaded_weight in weights:
478
- name, loaded_weight = self.maybe_remap_mistral(name, loaded_weight)
558
480
- if "rotary_emb.inv_freq" in name:
481
- continue
482
- if ("rotary_emb.cos_cached" in name
483
- or "rotary_emb.sin_cached" in name):
484
- # Models trained using ColossalAI may include these tensors in
485
- # the checkpoint. Skip them.
486
- continue
487
- # With tie_word_embeddings, we can skip lm_head.weight
488
- # The weight might appear unnecessarily in the files if the model is
489
- # processed with quantization, LoRA, fine-tuning, etc.
490
- if self.config.tie_word_embeddings and "lm_head.weight" in name:
491
- continue
492
- if scale_name := get_compressed_tensors_cache_scale(name):
493
- # Loading kv cache scales for compressed-tensors quantization
494
- param = params_dict[scale_name]
495
- weight_loader = getattr(param, "weight_loader",
496
- default_weight_loader)
497
- loaded_weight = loaded_weight[0]
498
- weight_loader(param, loaded_weight)
499
- continue
500
- for param_name, weight_name, shard_id in stacked_params_mapping:
501
- if weight_name not in name:
502
- continue
503
- name = name.replace(weight_name, param_name)
504
- # Skip loading extra bias for GPTQ models.
505
- if name.endswith(".bias") and name not in params_dict:
506
- continue
559
+ weights_group = group_weights_with_prefix(weights)
560
508
- if is_pp_missing_parameter(name, self):
509
- continue
561
+ self.model.load_weights(weights_group["model"])
562
511
- param = params_dict[name]
512
- weight_loader = param.weight_loader
513
- weight_loader(param, loaded_weight, shard_id)
514
-
515
- break
516
- else:
517
- # Skip loading extra bias for GPTQ models.
518
- if name.endswith(".bias") and name not in params_dict:
519
- continue
520
- # Remapping the name of FP8 kv-scale.
521
- name = maybe_remap_kv_scale_name(name, params_dict)
522
- if name is None:
523
- continue
524
-
525
- if is_pp_missing_parameter(name, self):
563
+ if not self.config.tie_word_embeddings:
564
+ lm_head_dict = dict(self.lm_head.named_parameters())
565
+ for name, loaded_weight in weights_group["lm_head"]:
566
+ if is_pp_missing_parameter(name, self.lm_head):
567
continue
568
528
- param = params_dict[name]
569
+ param = lm_head_dict[name]
570
weight_loader = getattr(param, "weight_loader",
571
default_weight_loader)
572
weight_loader(param, loaded_weight)
573
533
- # If this function is called, it should always initialize KV cache scale
534
- # factors (or else raise an exception). Thus, handled exceptions should
535
- # make sure to leave KV cache scale factors in a known good (dummy) state
574
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
537
- tp_size = get_tensor_model_parallel_world_size()
538
- tp_rank = get_tensor_model_parallel_rank()
539
- for layer_idx, scaling_factor in kv_cache_scales_loader(
540
- quantization_param_path, tp_rank, tp_size,
541
- self.config.num_hidden_layers,
542
- self.config.__class__.model_type):
543
- if not isinstance(self.model.layers[layer_idx], nn.Identity):
544
- layer_self_attn = self.model.layers[layer_idx].self_attn
545
-
546
- if is_hip():
547
- # The scaling factor convention we are assuming is
548
- # quantized_value * scaling_factor ~= true_value
549
- # which is consistent with the practice of setting
550
- # scaling_factor = tensor_amax / FPtype_max
551
- scaling_factor *= 2
552
- if hasattr(layer_self_attn, "kv_scale"):
553
- layer_self_attn.attn._kv_scale = scaling_factor
554
- else:
555
- raise RuntimeError("Self attention has no KV cache scaling "
556
- "factor attribute!")
575
+ self.model.load_kv_cache_scales(quantization_param_path)
576
577
# This function is used to remap the mistral format as
578
# used by Mistral and Llama <=2
579
def maybe_remap_mistral(
561
- self, name: str,
562
- loaded_weight: torch.Tensor) -> Tuple[str, torch.Tensor]:
580
+ self,
581
+ name: str,
582
+ loaded_weight: torch.Tensor,
583
+ ) -> Tuple[str, torch.Tensor]:
584
564
- def permute(w, n_heads):
585
+ def permute(w: torch.Tensor, n_heads: int):
586
attn_in = self.config.head_dim * n_heads
587
attn_out = self.config.hidden_size
588
vllm/model_executor/models/llama_embedding.py
+7
-46
@@ -5,13 +5,11 @@ from torch import nn
5
6
from vllm.attention import AttentionMetadata
7
from vllm.model_executor.layers.pooler import Pooler, PoolingType
8
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
9
-from vllm.model_executor.models.llama import LlamaModel
8
from vllm.model_executor.pooling_metadata import PoolingMetadata
9
from vllm.sequence import IntermediateTensors, PoolerOutput
10
11
from .interfaces import SupportsPP
14
-from .utils import is_pp_missing_parameter
12
+from .llama import LlamaModel
13
14
15
class LlamaEmbeddingModel(nn.Module, SupportsPP):
@@ -44,9 +42,8 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
42
intermediate_tensors: Optional[IntermediateTensors] = None,
43
inputs_embeds: Optional[torch.Tensor] = None,
44
) -> Union[torch.Tensor, IntermediateTensors]:
47
- return self.model.forward(input_ids, positions, kv_caches,
48
- attn_metadata, intermediate_tensors,
49
- inputs_embeds)
45
+ return self.model(input_ids, positions, kv_caches, attn_metadata,
46
+ intermediate_tensors, inputs_embeds)
47
48
def pooler(
49
self,
@@ -56,43 +53,7 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
53
return self._pooler(hidden_states, pooling_metadata)
54
55
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
59
- stacked_params_mapping = [
60
- # (param_name, shard_name, shard_id)
61
- ("qkv_proj", "q_proj", "q"),
62
- ("qkv_proj", "k_proj", "k"),
63
- ("qkv_proj", "v_proj", "v"),
64
- ("gate_up_proj", "gate_proj", 0),
65
- ("gate_up_proj", "up_proj", 1),
66
- ]
67
- params_dict = dict(self.model.named_parameters())
68
- for name, loaded_weight in weights:
69
- if "rotary_emb.inv_freq" in name:
70
- continue
71
- if ("rotary_emb.cos_cached" in name
72
- or "rotary_emb.sin_cached" in name):
73
- # Models trained using ColossalAI may include these tensors in
74
- # the checkpoint. Skip them.
75
- continue
76
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
77
- if weight_name not in name:
78
- continue
79
- name = name.replace(weight_name, param_name)
80
- # Skip loading extra bias for GPTQ models.
81
- if name.endswith(".bias") and name not in params_dict:
82
- continue
83
- if is_pp_missing_parameter(name, self):
84
- continue
85
- param = params_dict[name]
86
- weight_loader = param.weight_loader
87
- weight_loader(param, loaded_weight, shard_id)
88
- break
89
- else:
90
- # Skip loading extra bias for GPTQ models.
91
- if name.endswith(".bias") and name not in params_dict:
92
- continue
93
- if is_pp_missing_parameter(name, self):
94
- continue
95
- param = params_dict[name]
96
- weight_loader = getattr(param, "weight_loader",
97
- default_weight_loader)
98
- weight_loader(param, loaded_weight)
56
+ self.model.load_weights(weights)
57
+
58
+ def load_kv_cache_scales(self, quantization_param_path: str) -> None:
59
+ self.model.load_kv_cache_scales(quantization_param_path)
vllm/model_executor/models/qwen2.py
+53
-38
@@ -48,7 +48,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsLoRA, SupportsPP
51
-from .utils import (PPMissingLayer, is_pp_missing_parameter,
51
+from .utils import (PPMissingLayer, group_weights_with_prefix,
52
+ is_pp_missing_parameter,
53
make_empty_intermediate_tensors_factory, make_layers)
54
55
@@ -300,6 +301,47 @@ class Qwen2Model(nn.Module):
301
hidden_states, _ = self.norm(hidden_states, residual)
302
return hidden_states
303
304
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
305
+ stacked_params_mapping = [
306
+ # (param_name, shard_name, shard_id)
307
+ ("qkv_proj", "q_proj", "q"),
308
+ ("qkv_proj", "k_proj", "k"),
309
+ ("qkv_proj", "v_proj", "v"),
310
+ ("gate_up_proj", "gate_proj", 0),
311
+ ("gate_up_proj", "up_proj", 1),
312
+ ]
313
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
314
+ for name, loaded_weight in weights:
315
+ if "rotary_emb.inv_freq" in name:
316
+ continue
317
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
318
+ if weight_name not in name:
319
+ continue
320
+ name = name.replace(weight_name, param_name)
321
+ # Skip loading extra bias for GPTQ models.
322
+ if name.endswith(".bias") and name not in params_dict:
323
+ continue
324
+ if is_pp_missing_parameter(name, self):
325
+ continue
326
+ param = params_dict[name]
327
+ weight_loader = param.weight_loader
328
+ weight_loader(param, loaded_weight, shard_id)
329
+ break
330
+ else:
331
+ # Skip loading extra bias for GPTQ models.
332
+ if name.endswith(".bias") and name not in params_dict:
333
+ continue
334
+ # Remapping the name of FP8 kv-scale.
335
+ name = maybe_remap_kv_scale_name(name, params_dict)
336
+ if name is None:
337
+ continue
338
+ if is_pp_missing_parameter(name, self):
339
+ continue
340
+ param = params_dict[name]
341
+ weight_loader = getattr(param, "weight_loader",
342
+ default_weight_loader)
343
+ weight_loader(param, loaded_weight)
344
+
345
346
class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
347
packed_modules_mapping = {
@@ -393,44 +435,17 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
435
return next_tokens
436
437
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
396
- stacked_params_mapping = [
397
- # (param_name, shard_name, shard_id)
398
- ("qkv_proj", "q_proj", "q"),
399
- ("qkv_proj", "k_proj", "k"),
400
- ("qkv_proj", "v_proj", "v"),
401
- ("gate_up_proj", "gate_proj", 0),
402
- ("gate_up_proj", "up_proj", 1),
403
- ]
404
- params_dict = dict(self.named_parameters(remove_duplicate=False))
405
- for name, loaded_weight in weights:
406
- if "rotary_emb.inv_freq" in name:
407
- continue
408
- if self.config.tie_word_embeddings and "lm_head.weight" in name:
409
- continue
410
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
411
- if weight_name not in name:
412
- continue
413
- name = name.replace(weight_name, param_name)
414
- # Skip loading extra bias for GPTQ models.
415
- if name.endswith(".bias") and name not in params_dict:
416
- continue
417
- if is_pp_missing_parameter(name, self):
418
- continue
419
- param = params_dict[name]
420
- weight_loader = param.weight_loader
421
- weight_loader(param, loaded_weight, shard_id)
422
- break
423
- else:
424
- # Skip loading extra bias for GPTQ models.
425
- if name.endswith(".bias") and name not in params_dict:
426
- continue
427
- # Remapping the name of FP8 kv-scale.
428
- name = maybe_remap_kv_scale_name(name, params_dict)
429
- if name is None:
430
- continue
431
- if is_pp_missing_parameter(name, self):
438
+ weights_group = group_weights_with_prefix(weights)
439
+
440
+ self.model.load_weights(weights_group["model"])
441
+
442
+ if not self.config.tie_word_embeddings:
443
+ lm_head_dict = dict(self.lm_head.named_parameters())
444
+ for name, loaded_weight in weights_group["lm_head"]:
445
+ if is_pp_missing_parameter(name, self.lm_head):
446
continue
433
- param = params_dict[name]
447
+
448
+ param = lm_head_dict[name]
449
weight_loader = getattr(param, "weight_loader",
450
default_weight_loader)
451
weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen2_rm.py
+21
-51
@@ -4,7 +4,7 @@
4
# Copyright 2024 The Qwen team.
5
# Copyright 2023 The vLLM team.
6
"""Inference-only Qwen2-RM model compatible with HuggingFace weights."""
7
-from typing import Iterable, List, Optional, Tuple
7
+from typing import Iterable, List, Optional, Tuple, Union
8
9
import torch
10
from torch import nn
@@ -15,15 +15,14 @@ from vllm.config import CacheConfig, LoRAConfig
15
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
RowParallelLinear)
17
from vllm.model_executor.layers.pooler import Pooler, PoolingType
18
-from vllm.model_executor.layers.quantization.base_config import (
19
- QuantizationConfig)
20
-from vllm.model_executor.model_loader.weight_utils import (
21
- default_weight_loader, maybe_remap_kv_scale_name)
22
-from vllm.model_executor.models.qwen2 import Qwen2Model
18
+from vllm.model_executor.layers.quantization import QuantizationConfig
19
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
from vllm.model_executor.pooling_metadata import PoolingMetadata
21
from vllm.sequence import IntermediateTensors, PoolerOutput
22
26
-from .utils import is_pp_missing_parameter
23
+from .interfaces import SupportsPP
24
+from .qwen2 import Qwen2Model
25
+from .utils import group_weights_with_prefix
26
27
28
class ReLU(nn.Module):
@@ -37,7 +36,7 @@ class ReLU(nn.Module):
36
return self.activation(input)
37
38
40
-class Qwen2ForRewardModel(nn.Module):
39
+class Qwen2ForRewardModel(nn.Module, SupportsPP):
40
packed_modules_mapping = {
41
"qkv_proj": [
42
"q_proj",
@@ -97,6 +96,9 @@ class Qwen2ForRewardModel(nn.Module):
96
)
97
self._pooler = Pooler(pooling_type=PoolingType.ALL, normalize=False)
98
99
+ self.make_empty_intermediate_tensors = (
100
+ self.model.make_empty_intermediate_tensors)
101
+
102
def forward(
103
self,
104
input_ids: torch.Tensor,
@@ -104,7 +106,7 @@ class Qwen2ForRewardModel(nn.Module):
106
kv_caches: List[torch.Tensor],
107
attn_metadata: AttentionMetadata,
108
intermediate_tensors: Optional[IntermediateTensors] = None,
107
- ) -> torch.Tensor:
109
+ ) -> Union[torch.Tensor, IntermediateTensors]:
110
hidden_states = self.model(input_ids, positions, kv_caches,
111
attn_metadata, intermediate_tensors)
112
logits, _ = self.score(hidden_states)
@@ -118,45 +120,13 @@ class Qwen2ForRewardModel(nn.Module):
120
return self._pooler(hidden_states, pooling_metadata)
121
122
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
121
- stacked_params_mapping = [
122
- # (param_name, shard_name, shard_id)
123
- ("qkv_proj", "q_proj", "q"),
124
- ("qkv_proj", "k_proj", "k"),
125
- ("qkv_proj", "v_proj", "v"),
126
- ("gate_up_proj", "gate_proj", 0),
127
- ("gate_up_proj", "up_proj", 1),
128
- ]
129
- params_dict = dict(self.named_parameters(remove_duplicate=False))
130
- for name, loaded_weight in weights:
131
- # Skip loading lm_head for embedding model
132
- if name == "lm_head.weight":
133
- continue
134
- if "rotary_emb.inv_freq" in name:
135
- continue
136
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
137
- if weight_name not in name:
138
- continue
139
- name = name.replace(weight_name, param_name)
140
- # Skip loading extra bias for GPTQ models.
141
- if name.endswith(".bias") and name not in params_dict:
142
- continue
143
- if is_pp_missing_parameter(name, self):
144
- continue
145
- param = params_dict[name]
146
- weight_loader = param.weight_loader
147
- weight_loader(param, loaded_weight, shard_id)
148
- break
149
- else:
150
- # Skip loading extra bias for GPTQ models.
151
- if name.endswith(".bias") and name not in params_dict:
152
- continue
153
- # Remapping the name of FP8 kv-scale.
154
- name = maybe_remap_kv_scale_name(name, params_dict)
155
- if name is None:
156
- continue
157
- if is_pp_missing_parameter(name, self):
158
- continue
159
- param = params_dict[name]
160
- weight_loader = getattr(param, "weight_loader",
161
- default_weight_loader)
162
- weight_loader(param, loaded_weight)
123
+ weights_group = group_weights_with_prefix(weights)
124
+
125
+ self.model.load_weights(weights_group["model"])
126
+
127
+ score_dict = dict(self.score.named_parameters())
128
+ for name, loaded_weight in weights_group["score"]:
129
+ param = score_dict[name]
130
+ weight_loader = getattr(param, "weight_loader",
131
+ default_weight_loader)
132
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/utils.py
+6
-4
@@ -306,10 +306,12 @@ def get_pp_missing_layer_names(model: torch.nn.Module) -> List[str]:
306
307
def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool:
308
"""Check if a parameter is missing in a pipeline parallel model."""
309
- for missing_layer_name in get_pp_missing_layer_names(model):
310
- if name.startswith(missing_layer_name):
311
- return True
312
- return False
309
+ if isinstance(model, PPMissingLayer):
310
+ return True
311
+
312
+ return any(
313
+ name.startswith(missing_layer_name)
314
+ for missing_layer_name in get_pp_missing_layer_names(model))
315
316
317
def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
vllm/worker/embedding_model_runner.py
+41
-16
@@ -1,11 +1,12 @@
1
import dataclasses
2
-from typing import Any, Dict, List, Optional, Tuple, Type
2
+from typing import Any, Dict, List, Optional, Tuple, Type, Union
3
4
import torch
5
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
ModelConfig, ObservabilityConfig, ParallelConfig,
8
PromptAdapterConfig, SchedulerConfig)
9
+from vllm.distributed import get_pp_group
10
from vllm.forward_context import set_forward_context
11
from vllm.logger import init_logger
12
from vllm.model_executor.pooling_metadata import PoolingMetadata
@@ -66,7 +67,7 @@ class EmbeddingModelRunner(
67
kv_caches: List[torch.Tensor],
68
intermediate_tensors: Optional[IntermediateTensors] = None,
69
num_steps: int = 1,
69
- ) -> Optional[List[PoolerOutput]]:
70
+ ) -> Optional[Union[List[PoolerOutput], IntermediateTensors]]:
71
if num_steps > 1:
72
raise ValueError(
73
"EmbeddingModelRunner does not support multi-step execution.")
@@ -107,28 +108,52 @@ class EmbeddingModelRunner(
108
for _ in range(num_layers)
109
]
110
110
- execute_model_kwargs = {
111
- "input_ids":
112
- model_input.input_tokens,
113
- "positions":
114
- model_input.input_positions,
115
- "kv_caches":
116
- kv_caches,
117
- "attn_metadata":
118
- model_input.attn_metadata,
119
- **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
120
- device=self.device),
121
- }
111
+ multi_modal_kwargs = model_input.multi_modal_kwargs or {}
112
+ if (self.observability_config is not None
113
+ and self.observability_config.collect_model_forward_time):
114
+ model_forward_start = torch.cuda.Event(enable_timing=True)
115
+ model_forward_end = torch.cuda.Event(enable_timing=True)
116
+ model_forward_start.record()
117
118
with set_forward_context(model_input.attn_metadata):
124
- hidden_states = model_executable(**execute_model_kwargs)
119
+ hidden_or_intermediate_states = model_executable(
120
+ input_ids=model_input.input_tokens,
121
+ positions=model_input.input_positions,
122
+ kv_caches=kv_caches,
123
+ attn_metadata=model_input.attn_metadata,
124
+ intermediate_tensors=intermediate_tensors,
125
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs,
126
+ device=self.device))
127
+
128
+ if (self.observability_config is not None
129
+ and self.observability_config.collect_model_forward_time):
130
+ model_forward_end.record()
131
+
132
+ # Only perform pooling in the last pipeline stage.
133
+ if not get_pp_group().is_last_rank:
134
+ if (self.is_driver_worker
135
+ and hidden_or_intermediate_states is not None
136
+ and isinstance(hidden_or_intermediate_states,
137
+ IntermediateTensors)
138
+ and self.observability_config is not None
139
+ and self.observability_config.collect_model_forward_time):
140
+ model_forward_end.synchronize()
141
+ model_forward_time = model_forward_start.elapsed_time(
142
+ model_forward_end)
143
+ orig_model_forward_time = 0.0
144
+ if intermediate_tensors is not None:
145
+ orig_model_forward_time = intermediate_tensors.tensors.get(
146
+ "model_forward_time", torch.tensor(0.0)).item()
147
+ hidden_or_intermediate_states.tensors["model_forward_time"] = (
148
+ torch.tensor(model_forward_time + orig_model_forward_time))
149
+ return hidden_or_intermediate_states
150
151
# Only perform pooling in the driver worker.
152
if not self.is_driver_worker:
153
return []
154
155
return [
131
- self.model.pooler(hidden_states=hidden_states,
156
+ self.model.pooler(hidden_states=hidden_or_intermediate_states,
157
pooling_metadata=model_input.pooling_metadata)
158
]
159