A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix weight loading for Chameleon when TP>1 (#7410)
Cyrus Leung committed
Aug 13, 2024 at 13:33 UTC
7025b11d949b4efeb2584690c35f919c77027368
59 files changed
+414
-205
tests/conftest.py
+18
-8
@@ -4,7 +4,8 @@ import os
4
import sys
5
from collections import UserList
6
from enum import Enum
7
-from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union
7
+from typing import (Any, Callable, Dict, List, Optional, Tuple, TypedDict,
8
+ TypeVar, Union)
9
10
import pytest
11
import torch
@@ -27,7 +28,7 @@ from vllm.logger import init_logger
28
from vllm.outputs import RequestOutput
29
from vllm.sequence import SampleLogprobs
30
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
30
- is_cpu)
31
+ identity, is_cpu)
32
33
logger = init_logger(__name__)
34
@@ -197,6 +198,8 @@ class HfRunner:
198
is_embedding_model: bool = False,
199
is_vision_model: bool = False,
200
is_encoder_decoder_model: bool = False,
201
+ postprocess_inputs: Callable[[BatchEncoding],
202
+ BatchEncoding] = identity,
203
) -> None:
204
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
205
@@ -242,12 +245,14 @@ class HfRunner:
245
torch_dtype=torch_dtype,
246
trust_remote_code=True,
247
)
245
- except Exception:
248
+ except Exception as exc:
249
logger.warning(
247
- "Unable to auto-load processor from HuggingFace for "
248
- "model %s. Using tokenizer instead.", model_name)
250
+ "Unable to auto-load HuggingFace processor for model (%s). "
251
+ "Using tokenizer instead. Reason: %s", model_name, exc)
252
self.processor = self.tokenizer
253
254
+ self.postprocess_inputs = postprocess_inputs
255
+
256
def generate(
257
self,
258
prompts: List[str],
@@ -267,6 +272,7 @@ class HfRunner:
272
processor_kwargs["images"] = images[i]
273
274
inputs = self.processor(**processor_kwargs)
275
+ inputs = self.postprocess_inputs(inputs)
276
277
output_ids = self.model.generate(
278
**self.wrap_device(inputs),
@@ -336,6 +342,7 @@ class HfRunner:
342
processor_kwargs["images"] = images[i]
343
344
inputs = self.processor(**processor_kwargs)
345
+ inputs = self.postprocess_inputs(inputs)
346
347
output = self.model.generate(
348
**self.wrap_device(inputs),
@@ -420,6 +427,7 @@ class HfRunner:
427
processor_kwargs["images"] = images[i]
428
429
inputs = self.processor(**processor_kwargs)
430
+ inputs = self.postprocess_inputs(inputs)
431
432
output = self.model.generate(
433
**self.wrap_device(inputs),
@@ -552,7 +560,8 @@ class VllmRunner:
560
self,
561
prompts: List[str],
562
sampling_params: SamplingParams,
555
- images: Optional[List[Image.Image]] = None,
563
+ images: Optional[Union[List[Image.Image],
564
+ List[List[Image.Image]]]] = None,
565
) -> List[Tuple[List[List[int]], List[str]]]:
566
if images is not None:
567
assert len(prompts) == len(images)
@@ -587,7 +596,7 @@ class VllmRunner:
596
for req_output in req_outputs:
597
for sample in req_output.outputs:
598
output_str = sample.text
590
- output_ids = sample.token_ids
599
+ output_ids = list(sample.token_ids)
600
output_logprobs = sample.logprobs
601
outputs.append((output_ids, output_str, output_logprobs))
602
return outputs
@@ -596,7 +605,8 @@ class VllmRunner:
605
self,
606
prompts: List[str],
607
sampling_params: SamplingParams,
599
- images: Optional[List[Image.Image]] = None,
608
+ images: Optional[Union[List[Image.Image],
609
+ List[List[Image.Image]]]] = None,
610
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
611
assert sampling_params.logprobs is not None
612
tests/distributed/test_multimodal_broadcast.py
+4
@@ -18,8 +18,10 @@ from ..utils import fork_new_process_for_each_test
18
@pytest.mark.parametrize("model, distributed_executor_backend", [
19
("llava-hf/llava-1.5-7b-hf", "ray"),
20
("llava-hf/llava-v1.6-mistral-7b-hf", "ray"),
21
+ ("facebook/chameleon-7b", "ray"),
22
("llava-hf/llava-1.5-7b-hf", "mp"),
23
("llava-hf/llava-v1.6-mistral-7b-hf", "mp"),
24
+ ("facebook/chameleon-7b", "mp"),
25
])
26
@fork_new_process_for_each_test
27
def test_models(hf_runner, vllm_runner, image_assets, model: str,
@@ -34,6 +36,8 @@ def test_models(hf_runner, vllm_runner, image_assets, model: str,
36
from ..models.test_llava import models, run_test
37
elif model.startswith("llava-hf/llava-v1.6"):
38
from ..models.test_llava_next import models, run_test
39
+ elif model.startswith("facebook/chameleon"):
40
+ from ..models.test_chameleon import models, run_test
41
else:
42
raise NotImplementedError(f"Unsupported model: {model}")
43
tests/entrypoints/openai/test_oot_registration.py
+6
-2
@@ -1,5 +1,6 @@
1
import sys
2
import time
3
+from typing import Optional
4
5
import torch
6
from openai import OpenAI, OpenAIError
@@ -17,8 +18,11 @@ assert chatml_jinja_path.exists()
18
19
class MyOPTForCausalLM(OPTForCausalLM):
20
20
- def compute_logits(self, hidden_states: torch.Tensor,
21
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
21
+ def compute_logits(
22
+ self,
23
+ hidden_states: torch.Tensor,
24
+ sampling_metadata: SamplingMetadata,
25
+ ) -> Optional[torch.Tensor]:
26
# this dummy model always predicts the first token
27
logits = super().compute_logits(hidden_states, sampling_metadata)
28
logits.zero_()
tests/models/test_chameleon.py
+58
-33
@@ -1,11 +1,13 @@
1
-import re
1
from typing import List, Optional, Type
2
3
import pytest
4
+from transformers import BatchEncoding
5
6
from vllm.multimodal.utils import rescale_image_size
7
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
8
8
-from ..conftest import IMAGE_ASSETS, VllmRunner, _ImageAssets
9
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
+from .utils import check_outputs_equal
11
12
pytestmark = pytest.mark.vlm
13
@@ -19,9 +21,8 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21
models = ["facebook/chameleon-7b"]
22
23
22
-#TODO (ywang96): Add correctness test when chameleon is
23
-# available on transformers.
24
def run_test(
25
+ hf_runner: Type[HfRunner],
26
vllm_runner: Type[VllmRunner],
27
image_assets: _ImageAssets,
28
model: str,
@@ -29,13 +30,20 @@ def run_test(
30
size_factors: List[float],
31
dtype: str,
32
max_tokens: int,
33
+ num_logprobs: int,
34
tensor_parallel_size: int,
35
distributed_executor_backend: Optional[str] = None,
36
):
35
- """Test if the model can generate text given
36
- a batch of images and prompts.
37
-
37
+ """Inference result should be the same between hf and vllm.
38
+
39
+ All the image fixtures for the test is under tests/images.
40
+ For huggingface runner, we provide the PIL images as input.
41
+ For vllm runner, we provide MultiModalDataDict objects
42
+ and corresponding vision language config as input.
43
+ Note, the text input is also adjusted to abide by vllm contract.
44
+ The text output is sanitized to be able to compare with hf.
45
"""
46
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
47
images = [asset.pil_image for asset in image_assets]
48
49
inputs_per_image = [(
@@ -50,35 +58,49 @@ def run_test(
58
distributed_executor_backend=distributed_executor_backend,
59
enforce_eager=True) as vllm_model:
60
53
- for prompts, images in inputs_per_image:
54
- vllm_outputs = vllm_model.generate_greedy(prompts,
55
- max_tokens,
56
- images=images)
57
- for i in range(len(vllm_outputs)):
58
-
59
- # format prompt back to original
60
- replacements = {
61
- "<racm3:break>": "",
62
- "<eoss>": "",
63
- "<reserved08706>": ""
64
- }
65
- pattern = '|'.join(replacements.keys())
66
- vllm_result = re.sub(
67
- pattern,
68
- lambda match: replacements[match.group(0)], #noqa B023
69
- vllm_outputs[i][1])
70
- vllm_result = vllm_result.replace("<image>", "", 1023)
71
- assert vllm_result[:len(prompts[i])] == prompts[i]
72
-
73
- # assert at least 10 new characters are generated
74
- # (to take stop token into account)
75
- assert len(vllm_outputs[i][1]) - len(prompts[i]) > 10
61
+ vllm_outputs_per_image = [
62
+ vllm_model.generate_greedy_logprobs(prompts,
63
+ max_tokens,
64
+ num_logprobs=num_logprobs,
65
+ images=images)
66
+ for prompts, images in inputs_per_image
67
+ ]
68
+
69
+ def process(hf_inputs: BatchEncoding):
70
+ hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
71
+ .to(torch_dtype) # type: ignore
72
+ return hf_inputs
73
+
74
+ with hf_runner(model,
75
+ dtype=dtype,
76
+ postprocess_inputs=process,
77
+ is_vision_model=True) as hf_model:
78
+ hf_outputs_per_image = [
79
+ hf_model.generate_greedy_logprobs_limit(prompts,
80
+ max_tokens,
81
+ num_logprobs=num_logprobs,
82
+ images=images)
83
+ for prompts, images in inputs_per_image
84
+ ]
85
+
86
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
87
+ vllm_outputs_per_image):
88
+ # HF Logprobs include image tokens, unlike vLLM, so we don't directly
89
+ # compare them
90
+ check_outputs_equal(
91
+ outputs_0_lst=[outputs[:2] for outputs in hf_outputs],
92
+ outputs_1_lst=[outputs[:2] for outputs in vllm_outputs],
93
+ name_0="hf",
94
+ name_1="vllm",
95
+ )
96
97
98
@pytest.mark.parametrize("model", models)
99
@pytest.mark.parametrize(
100
"size_factors",
101
[
102
+ # No image
103
+ [],
104
# Single-scale
105
[1.0],
106
# Single-scale, batched
@@ -88,15 +110,18 @@ def run_test(
110
],
111
)
112
@pytest.mark.parametrize("dtype", ["bfloat16"])
91
-@pytest.mark.parametrize("max_tokens", [128])
92
-def test_models(vllm_runner, image_assets, model, size_factors, dtype: str,
93
- max_tokens: int) -> None:
113
+@pytest.mark.parametrize("max_tokens", [8])
114
+@pytest.mark.parametrize("num_logprobs", [5])
115
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
116
+ dtype, max_tokens, num_logprobs) -> None:
117
run_test(
118
+ hf_runner,
119
vllm_runner,
120
image_assets,
121
model,
122
size_factors=size_factors,
123
dtype=dtype,
124
max_tokens=max_tokens,
125
+ num_logprobs=num_logprobs,
126
tensor_parallel_size=1,
127
)
tests/models/test_llava.py
+13
-8
@@ -1,7 +1,7 @@
1
from typing import List, Optional, Tuple, Type
2
3
import pytest
4
-from transformers import AutoConfig, AutoTokenizer
4
+from transformers import AutoConfig, AutoTokenizer, BatchEncoding
5
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
@@ -110,16 +110,21 @@ def run_test(
110
for prompts, images in inputs_per_image
111
]
112
113
- with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
114
- if mantis_processor is not None:
113
+ if mantis_processor is not None:
114
116
- def process(*args, **kwargs):
117
- output = mantis_processor(*args, **kwargs)
118
- output["pixel_values"] = output["pixel_values"].to(torch_dtype)
119
- return output
115
+ def process(hf_inputs: BatchEncoding):
116
+ hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
117
+ .to(torch_dtype) # type: ignore
118
+ return hf_inputs
119
+ else:
120
121
- hf_model.processor = process
121
+ def process(hf_inputs: BatchEncoding):
122
+ return hf_inputs
123
124
+ with hf_runner(model,
125
+ dtype=dtype,
126
+ postprocess_inputs=process,
127
+ is_vision_model=True) as hf_model:
128
hf_outputs_per_image = [
129
hf_model.generate_greedy_logprobs_limit(prompts,
130
max_tokens,
tests/models/test_minicpmv.py
+9
-24
@@ -1,10 +1,9 @@
1
-from collections import UserDict
1
from typing import List, Optional, Tuple, Type
2
3
import pytest
4
import torch
5
import torch.types
7
-from transformers import BatchFeature
6
+from transformers import BatchEncoding
7
8
from vllm.multimodal.utils import rescale_image_size
9
from vllm.sequence import SampleLogprobs
@@ -14,18 +13,6 @@ from .utils import check_logprobs_close
13
14
pytestmark = pytest.mark.vlm
15
17
-
18
-class NestedInputs(UserDict):
19
-
20
- def __init__(self, model_inputs: BatchFeature):
21
- super().__init__({"model_inputs": model_inputs})
22
-
23
- self.model_inputs = model_inputs
24
-
25
- def to(self, device: torch.types.Device):
26
- return NestedInputs(self.model_inputs.to(device))
27
-
28
-
16
# The image token is placed before "user" on purpose so that the test can pass
17
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18
"stop_sign":
@@ -41,6 +28,10 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
28
models = ["openbmb/MiniCPM-Llama3-V-2_5"]
29
30
31
+def _wrap_inputs(hf_inputs: BatchEncoding) -> BatchEncoding:
32
+ return BatchEncoding({"model_inputs": hf_inputs})
33
+
34
+
35
def trunc_hf_output(hf_output: Tuple[List[int], str,
36
Optional[SampleLogprobs]]):
37
output_ids, output_str, out_logprobs = hf_output
@@ -105,11 +96,8 @@ def run_test(
96
for prompts, images in inputs_per_image
97
]
98
108
- with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
109
- hf_processor = hf_model.processor
110
- hf_model.processor = lambda **kw: NestedInputs(
111
- hf_processor(**kw) # type: ignore
112
- )
99
+ hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
100
+ with hf_model, torch.no_grad():
101
hf_outputs_per_image = [
102
hf_model.generate_greedy_logprobs_limit(prompts,
103
max_tokens,
@@ -224,11 +212,8 @@ def run_multi_image_test(
212
for prompts, images in inputs_per_case
213
]
214
227
- with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
228
- hf_processor = hf_model.processor
229
- hf_model.processor = lambda **kw: NestedInputs(
230
- hf_processor(**kw) # type: ignore
231
- )
215
+ hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
216
+ with hf_model, torch.no_grad():
217
hf_outputs_per_case = [
218
hf_model.generate_greedy_logprobs_limit(prompts,
219
max_tokens,
tests/models/test_oot_registration.py
+7
-2
@@ -1,3 +1,5 @@
1
+from typing import Optional
2
+
3
import torch
4
5
from vllm import LLM, ModelRegistry, SamplingParams
@@ -7,8 +9,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
9
10
class MyOPTForCausalLM(OPTForCausalLM):
11
10
- def compute_logits(self, hidden_states: torch.Tensor,
11
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
12
+ def compute_logits(
13
+ self,
14
+ hidden_states: torch.Tensor,
15
+ sampling_metadata: SamplingMetadata,
16
+ ) -> Optional[torch.Tensor]:
17
# this dummy model always predicts the first token
18
logits = super().compute_logits(hidden_states, sampling_metadata)
19
logits.zero_()
vllm/distributed/communication_op.py
+1
-1
@@ -19,7 +19,7 @@ def tensor_model_parallel_all_gather(input_: torch.Tensor,
19
20
def tensor_model_parallel_gather(input_: torch.Tensor,
21
dst: int = 0,
22
- dim: int = -1) -> torch.Tensor:
22
+ dim: int = -1) -> Optional[torch.Tensor]:
23
"""Gather the input tensor across model parallel group."""
24
return get_tp_group().gather(input_, dst, dim)
25
vllm/distributed/parallel_state.py
+1
-1
@@ -329,7 +329,7 @@ class GroupCoordinator:
329
def gather(self,
330
input_: torch.Tensor,
331
dst: int = 0,
332
- dim: int = -1) -> torch.Tensor:
332
+ dim: int = -1) -> Optional[torch.Tensor]:
333
"""
334
NOTE: We assume that the input tensor is on the same device across
335
all the ranks.
vllm/model_executor/layers/logits_processor.py
+8
-4
@@ -50,7 +50,7 @@ class LogitsProcessor(nn.Module):
50
hidden_states: torch.Tensor,
51
sampling_metadata: SamplingMetadata,
52
embedding_bias: Optional[torch.Tensor] = None,
53
- ) -> torch.Tensor:
53
+ ) -> Optional[torch.Tensor]:
54
if self.logits_as_input:
55
logits = hidden_states
56
else:
@@ -73,14 +73,18 @@ class LogitsProcessor(nn.Module):
73
74
return logits
75
76
- def _get_logits(self, hidden_states: torch.Tensor,
77
- lm_head: VocabParallelEmbedding,
78
- embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
76
+ def _get_logits(
77
+ self,
78
+ hidden_states: torch.Tensor,
79
+ lm_head: VocabParallelEmbedding,
80
+ embedding_bias: Optional[torch.Tensor],
81
+ ) -> Optional[torch.Tensor]:
82
# Get the logits for the next tokens.
83
logits = lm_head.linear_method.apply(lm_head,
84
hidden_states,
85
bias=embedding_bias)
86
if self.use_gather:
87
+ # None may be returned for rank > 0
88
logits = tensor_model_parallel_gather(logits)
89
else:
90
# Gather is not supported for some devices such as TPUs.
vllm/model_executor/model_loader/weight_utils.py
+25
-2
@@ -19,6 +19,7 @@ from tqdm.auto import tqdm
19
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
20
21
from vllm.config import LoadConfig, ModelConfig
22
+from vllm.distributed import get_tensor_model_parallel_rank
23
from vllm.logger import init_logger
24
from vllm.model_executor.layers.quantization import (QuantizationConfig,
25
get_quantization_config)
@@ -514,8 +515,30 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
515
def default_weight_loader(param: torch.Tensor,
516
loaded_weight: torch.Tensor) -> None:
517
"""Default weight loader."""
517
- assert param.size() == loaded_weight.size()
518
- param.data.copy_(loaded_weight)
518
+ try:
519
+ assert param.size() == loaded_weight.size(), (
520
+ f"Attempted to load weight ({loaded_weight.size()}) "
521
+ f"into parameter ({param.size()})")
522
+
523
+ param.data.copy_(loaded_weight)
524
+ except Exception:
525
+ # NOTE: This exception is added for the purpose of setting breakpoint to
526
+ # debug weight loading issues.
527
+ raise
528
+
529
+
530
+def row_parallel_weight_loader(param: torch.Tensor,
531
+ loaded_weight: torch.Tensor) -> None:
532
+ """Load weights that are row-parallelized."""
533
+ tp_rank = get_tensor_model_parallel_rank()
534
+ shard_dim = 0 if param.dim() != 1 else None
535
+
536
+ if shard_dim is not None:
537
+ shard_size = param.data.shape[shard_dim]
538
+ start_idx = tp_rank * shard_size
539
+ loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size)
540
+
541
+ return default_weight_loader(param, loaded_weight)
542
543
544
def initialize_dummy_weights(
vllm/model_executor/models/arctic.py
+5
-2
@@ -433,8 +433,11 @@ class ArcticForCausalLM(nn.Module):
433
attn_metadata)
434
return hidden_states
435
436
- def compute_logits(self, hidden_states: torch.Tensor,
437
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
436
+ def compute_logits(
437
+ self,
438
+ hidden_states: torch.Tensor,
439
+ sampling_metadata: SamplingMetadata,
440
+ ) -> Optional[torch.Tensor]:
441
logits = self.logits_processor(self.lm_head, hidden_states,
442
sampling_metadata)
443
return logits
vllm/model_executor/models/baichuan.py
+5
-2
@@ -346,8 +346,11 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
346
attn_metadata)
347
return hidden_states
348
349
- def compute_logits(self, hidden_states: torch.Tensor,
350
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
349
+ def compute_logits(
350
+ self,
351
+ hidden_states: torch.Tensor,
352
+ sampling_metadata: SamplingMetadata,
353
+ ) -> Optional[torch.Tensor]:
354
logits = self.logits_processor(self.lm_head, hidden_states,
355
sampling_metadata)
356
return logits
vllm/model_executor/models/bart.py
+5
-2
@@ -872,8 +872,11 @@ class BartForConditionalGeneration(nn.Module):
872
return self.model(input_ids, positions, encoder_input_ids,
873
encoder_positions, kv_caches, attn_metadata)
874
875
- def compute_logits(self, hidden_states: torch.Tensor,
876
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
875
+ def compute_logits(
876
+ self,
877
+ hidden_states: torch.Tensor,
878
+ sampling_metadata: SamplingMetadata,
879
+ ) -> Optional[torch.Tensor]:
880
logits = self.logits_processor(self.lm_head, hidden_states,
881
sampling_metadata)
882
return logits
vllm/model_executor/models/blip2.py
+5
-2
@@ -637,8 +637,11 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
637
638
return hidden_states
639
640
- def compute_logits(self, hidden_states: torch.Tensor,
641
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
640
+ def compute_logits(
641
+ self,
642
+ hidden_states: torch.Tensor,
643
+ sampling_metadata: SamplingMetadata,
644
+ ) -> Optional[torch.Tensor]:
645
logits = self.logits_processor(self.get_lm_head(), hidden_states,
646
sampling_metadata)
647
return logits
vllm/model_executor/models/bloom.py
+5
-2
@@ -292,8 +292,11 @@ class BloomForCausalLM(nn.Module):
292
attn_metadata)
293
return hidden_states
294
295
- def compute_logits(self, hidden_states: torch.Tensor,
296
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
295
+ def compute_logits(
296
+ self,
297
+ hidden_states: torch.Tensor,
298
+ sampling_metadata: SamplingMetadata,
299
+ ) -> Optional[torch.Tensor]:
300
logits = self.logits_processor(self.lm_head, hidden_states,
301
sampling_metadata)
302
return logits
vllm/model_executor/models/chameleon.py
+18
-5
@@ -25,8 +25,10 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
25
from vllm.model_executor.layers.sampler import Sampler
26
from vllm.model_executor.layers.vocab_parallel_embedding import (
27
ParallelLMHead, VocabParallelEmbedding)
28
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28
+from vllm.model_executor.model_loader.weight_utils import (
29
+ default_weight_loader, row_parallel_weight_loader)
30
from vllm.model_executor.sampling_metadata import SamplingMetadata
31
+from vllm.model_executor.utils import set_weight_attrs
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.image import (cached_get_tokenizer,
34
repeat_and_pad_image_tokens)
@@ -141,6 +143,11 @@ class ChameleonLayerNorm(nn.LayerNorm):
143
super().__init__(hidden_size, *args, **kwargs)
144
self.normalized_shape = (hidden_size[-1], )
145
146
+ set_weight_attrs(self.weight,
147
+ {"weight_loader": row_parallel_weight_loader})
148
+ set_weight_attrs(self.bias,
149
+ {"weight_loader": row_parallel_weight_loader})
150
+
151
def forward(self, hidden_states):
152
hidden_states = F.layer_norm(hidden_states,
153
self.normalized_shape,
@@ -697,6 +704,8 @@ class ChameleonVQVAEEncoder(nn.Module):
704
)
705
706
def forward(self, pixel_values: torch.Tensor):
707
+ pixel_values = pixel_values.to(self.conv_in.weight.dtype)
708
+
709
# downsampling
710
hidden_states = [self.conv_in(pixel_values)]
711
for i_level in range(self.num_resolutions):
@@ -959,15 +968,19 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
968
attn_metadata)
969
return hidden_states
970
962
- def compute_logits(self, hidden_states: torch.Tensor,
963
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
971
+ def compute_logits(
972
+ self,
973
+ hidden_states: torch.Tensor,
974
+ sampling_metadata: SamplingMetadata,
975
+ ) -> Optional[torch.Tensor]:
976
logits = self.logits_processor(self.lm_head, hidden_states,
977
sampling_metadata)
978
979
# Disallow image tokens which does not include special
980
# begin-image and end-image tokens
969
- image_tokens = self.model.vocabulary_mapping.image_tokens
970
- logits[:, image_tokens] = torch.finfo(logits.dtype).min
981
+ if logits is not None:
982
+ image_tokens = self.model.vocabulary_mapping.image_tokens
983
+ logits[:, image_tokens] = torch.finfo(logits.dtype).min
984
985
return logits
986
vllm/model_executor/models/chatglm.py
+5
-2
@@ -372,8 +372,11 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
372
attn_metadata)
373
return hidden_states
374
375
- def compute_logits(self, hidden_states: torch.Tensor,
376
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
375
+ def compute_logits(
376
+ self,
377
+ hidden_states: torch.Tensor,
378
+ sampling_metadata: SamplingMetadata,
379
+ ) -> Optional[torch.Tensor]:
380
logits = self.logits_processor(self.lm_head, hidden_states,
381
sampling_metadata)
382
return logits
vllm/model_executor/models/commandr.py
+10
-19
@@ -25,13 +25,11 @@ from typing import Iterable, List, Optional, Set, Tuple
25
import torch
26
import torch.utils.checkpoint
27
from torch import nn
28
-from torch.nn.parameter import Parameter
28
from transformers import CohereConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import CacheConfig, LoRAConfig
33
-from vllm.distributed import (get_tensor_model_parallel_rank,
34
- get_tensor_model_parallel_world_size)
32
+from vllm.distributed import get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
35
QKVParallelLinear,
@@ -43,7 +41,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
VocabParallelEmbedding)
46
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
+from vllm.model_executor.model_loader.weight_utils import (
45
+ default_weight_loader, row_parallel_weight_loader)
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.model_executor.utils import set_weight_attrs
48
from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -67,25 +66,14 @@ class LayerNorm(nn.Module):
66
super().__init__()
67
self.weight = nn.Parameter(torch.ones(param_shape))
68
self.variance_epsilon = eps
70
- set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
69
+ set_weight_attrs(self.weight,
70
+ {"weight_loader": row_parallel_weight_loader})
71
72
def forward(self, hidden_states, residuals=None):
73
hidden_states = layer_norm_func(hidden_states, self.weight,
74
self.variance_epsilon)
75
return hidden_states, residuals
76
77
- def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
78
- tp_rank = get_tensor_model_parallel_rank()
79
- shard_dim = 0 if param.dim() != 1 else None
80
- param_data = param.data
81
- if shard_dim is not None:
82
- shard_size = param_data.shape[shard_dim]
83
- start_idx = tp_rank * shard_size
84
- loaded_weight = loaded_weight.narrow(shard_dim, start_idx,
85
- shard_size)
86
- assert param_data.shape == loaded_weight.shape
87
- param_data.copy_(loaded_weight)
88
-
77
78
# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
79
class CohereMLP(nn.Module):
@@ -359,8 +347,11 @@ class CohereForCausalLM(nn.Module):
347
attn_metadata)
348
return hidden_states
349
362
- def compute_logits(self, hidden_states: torch.Tensor,
363
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
350
+ def compute_logits(
351
+ self,
352
+ hidden_states: torch.Tensor,
353
+ sampling_metadata: SamplingMetadata,
354
+ ) -> Optional[torch.Tensor]:
355
is_not_lora = hasattr(self.model.embed_tokens, 'weight')
356
if is_not_lora:
357
logits = self.logits_processor(self.model.embed_tokens,
vllm/model_executor/models/dbrx.py
+5
-2
@@ -388,8 +388,11 @@ class DbrxForCausalLM(nn.Module):
388
attn_metadata)
389
return hidden_states
390
391
- def compute_logits(self, hidden_states: torch.Tensor,
392
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
391
+ def compute_logits(
392
+ self,
393
+ hidden_states: torch.Tensor,
394
+ sampling_metadata: SamplingMetadata,
395
+ ) -> Optional[torch.Tensor]:
396
logits = self.logits_processor(self.lm_head, hidden_states,
397
sampling_metadata)
398
return logits
vllm/model_executor/models/deepseek.py
+5
-2
@@ -395,8 +395,11 @@ class DeepseekForCausalLM(nn.Module):
395
attn_metadata)
396
return hidden_states
397
398
- def compute_logits(self, hidden_states: torch.Tensor,
399
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
398
+ def compute_logits(
399
+ self,
400
+ hidden_states: torch.Tensor,
401
+ sampling_metadata: SamplingMetadata,
402
+ ) -> Optional[torch.Tensor]:
403
logits = self.logits_processor(self.lm_head, hidden_states,
404
sampling_metadata)
405
return logits
vllm/model_executor/models/deepseek_v2.py
+5
-2
@@ -505,8 +505,11 @@ class DeepseekV2ForCausalLM(nn.Module):
505
attn_metadata, intermediate_tensors)
506
return hidden_states
507
508
- def compute_logits(self, hidden_states: torch.Tensor,
509
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
508
+ def compute_logits(
509
+ self,
510
+ hidden_states: torch.Tensor,
511
+ sampling_metadata: SamplingMetadata,
512
+ ) -> Optional[torch.Tensor]:
513
logits = self.logits_processor(self.lm_head, hidden_states,
514
sampling_metadata)
515
return logits
vllm/model_executor/models/falcon.py
+5
-2
@@ -420,8 +420,11 @@ class FalconForCausalLM(nn.Module):
420
)
421
return hidden_states
422
423
- def compute_logits(self, hidden_states: torch.Tensor,
424
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
423
+ def compute_logits(
424
+ self,
425
+ hidden_states: torch.Tensor,
426
+ sampling_metadata: SamplingMetadata,
427
+ ) -> Optional[torch.Tensor]:
428
logits = self.logits_processor(self.lm_head, hidden_states,
429
sampling_metadata)
430
return logits
vllm/model_executor/models/fuyu.py
+5
-2
@@ -287,8 +287,11 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
287
)
288
return hidden_states
289
290
- def compute_logits(self, hidden_states: torch.Tensor,
291
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
290
+ def compute_logits(
291
+ self,
292
+ hidden_states: torch.Tensor,
293
+ sampling_metadata: SamplingMetadata,
294
+ ) -> Optional[torch.Tensor]:
295
logits = self.language_model.logits_processor(
296
self.language_model.lm_head, hidden_states, sampling_metadata)
297
return logits
vllm/model_executor/models/gemma.py
+5
-2
@@ -352,8 +352,11 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
352
attn_metadata)
353
return hidden_states
354
355
- def compute_logits(self, hidden_states: torch.Tensor,
356
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
355
+ def compute_logits(
356
+ self,
357
+ hidden_states: torch.Tensor,
358
+ sampling_metadata: SamplingMetadata,
359
+ ) -> Optional[torch.Tensor]:
360
logits = self.logits_processor(self.model.embed_tokens, hidden_states,
361
sampling_metadata)
362
return logits
vllm/model_executor/models/gemma2.py
+5
-2
@@ -343,8 +343,11 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
343
attn_metadata)
344
return hidden_states
345
346
- def compute_logits(self, hidden_states: torch.Tensor,
347
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
346
+ def compute_logits(
347
+ self,
348
+ hidden_states: torch.Tensor,
349
+ sampling_metadata: SamplingMetadata,
350
+ ) -> Optional[torch.Tensor]:
351
logits = self.logits_processor(self.model.embed_tokens, hidden_states,
352
sampling_metadata)
353
return logits
vllm/model_executor/models/gpt2.py
+5
-2
@@ -265,8 +265,11 @@ class GPT2LMHeadModel(nn.Module):
265
attn_metadata, intermediate_tensors)
266
return hidden_states
267
268
- def compute_logits(self, hidden_states: torch.Tensor,
269
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
268
+ def compute_logits(
269
+ self,
270
+ hidden_states: torch.Tensor,
271
+ sampling_metadata: SamplingMetadata,
272
+ ) -> Optional[torch.Tensor]:
273
logits = self.logits_processor(self.lm_head, hidden_states,
274
sampling_metadata)
275
return logits
vllm/model_executor/models/gpt_bigcode.py
+5
-2
@@ -279,8 +279,11 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
279
attn_metadata)
280
return hidden_states
281
282
- def compute_logits(self, hidden_states: torch.Tensor,
283
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
282
+ def compute_logits(
283
+ self,
284
+ hidden_states: torch.Tensor,
285
+ sampling_metadata: SamplingMetadata,
286
+ ) -> Optional[torch.Tensor]:
287
logits = self.logits_processor(self.lm_head, hidden_states,
288
sampling_metadata)
289
return logits
vllm/model_executor/models/gpt_j.py
+5
-2
@@ -246,8 +246,11 @@ class GPTJForCausalLM(nn.Module):
246
attn_metadata)
247
return hidden_states
248
249
- def compute_logits(self, hidden_states: torch.Tensor,
250
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
249
+ def compute_logits(
250
+ self,
251
+ hidden_states: torch.Tensor,
252
+ sampling_metadata: SamplingMetadata,
253
+ ) -> Optional[torch.Tensor]:
254
logits = self.logits_processor(self.lm_head, hidden_states,
255
sampling_metadata, self.lm_head.bias)
256
return logits
vllm/model_executor/models/gpt_neox.py
+5
-2
@@ -258,8 +258,11 @@ class GPTNeoXForCausalLM(nn.Module):
258
attn_metadata)
259
return hidden_states
260
261
- def compute_logits(self, hidden_states: torch.Tensor,
262
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
261
+ def compute_logits(
262
+ self,
263
+ hidden_states: torch.Tensor,
264
+ sampling_metadata: SamplingMetadata,
265
+ ) -> Optional[torch.Tensor]:
266
logits = self.logits_processor(self.embed_out, hidden_states,
267
sampling_metadata)
268
return logits
vllm/model_executor/models/internlm2.py
+5
-2
@@ -279,8 +279,11 @@ class InternLM2ForCausalLM(nn.Module):
279
attn_metadata)
280
return hidden_states
281
282
- def compute_logits(self, hidden_states: torch.Tensor,
283
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
282
+ def compute_logits(
283
+ self,
284
+ hidden_states: torch.Tensor,
285
+ sampling_metadata: SamplingMetadata,
286
+ ) -> Optional[torch.Tensor]:
287
logits = self.logits_processor(self.output, hidden_states,
288
sampling_metadata)
289
return logits
vllm/model_executor/models/internvl.py
+5
-2
@@ -466,8 +466,11 @@ class InternVLChatModel(nn.Module, SupportsVision):
466
inputs_embeds=inputs_embeds)
467
return hidden_states
468
469
- def compute_logits(self, hidden_states: torch.Tensor,
470
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
469
+ def compute_logits(
470
+ self,
471
+ hidden_states: torch.Tensor,
472
+ sampling_metadata: SamplingMetadata,
473
+ ) -> Optional[torch.Tensor]:
474
return self.language_model.compute_logits(hidden_states,
475
sampling_metadata)
476
vllm/model_executor/models/jais.py
+5
-2
@@ -295,8 +295,11 @@ class JAISLMHeadModel(nn.Module):
295
attn_metadata)
296
return hidden_states
297
298
- def compute_logits(self, hidden_states: torch.Tensor,
299
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
298
+ def compute_logits(
299
+ self,
300
+ hidden_states: torch.Tensor,
301
+ sampling_metadata: SamplingMetadata,
302
+ ) -> Optional[torch.Tensor]:
303
logits = self.logits_processor(self.lm_head, hidden_states,
304
sampling_metadata)
305
return logits
vllm/model_executor/models/jamba.py
+5
-2
@@ -861,8 +861,11 @@ class JambaForCausalLM(nn.Module, HasInnerState):
861
dtype=dtype,
862
device="cuda"))
863
864
- def compute_logits(self, hidden_states: torch.Tensor,
865
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
864
+ def compute_logits(
865
+ self,
866
+ hidden_states: torch.Tensor,
867
+ sampling_metadata: SamplingMetadata,
868
+ ) -> Optional[torch.Tensor]:
869
logits = self.logits_processor(self.lm_head, hidden_states,
870
sampling_metadata)
871
return logits
vllm/model_executor/models/llama.py
+5
-2
@@ -430,8 +430,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
430
attn_metadata, intermediate_tensors)
431
return model_output
432
433
- def compute_logits(self, hidden_states: torch.Tensor,
434
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
433
+ def compute_logits(
434
+ self,
435
+ hidden_states: torch.Tensor,
436
+ sampling_metadata: SamplingMetadata,
437
+ ) -> Optional[torch.Tensor]:
438
logits = self.logits_processor(self.lm_head, hidden_states,
439
sampling_metadata)
440
return logits
vllm/model_executor/models/llava.py
+5
-2
@@ -355,8 +355,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
355
356
return hidden_states
357
358
- def compute_logits(self, hidden_states: torch.Tensor,
359
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
358
+ def compute_logits(
359
+ self,
360
+ hidden_states: torch.Tensor,
361
+ sampling_metadata: SamplingMetadata,
362
+ ) -> Optional[torch.Tensor]:
363
return self.language_model.compute_logits(hidden_states,
364
sampling_metadata)
365
vllm/model_executor/models/llava_next.py
+5
-2
@@ -588,8 +588,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
588
589
return hidden_states
590
591
- def compute_logits(self, hidden_states: torch.Tensor,
592
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
591
+ def compute_logits(
592
+ self,
593
+ hidden_states: torch.Tensor,
594
+ sampling_metadata: SamplingMetadata,
595
+ ) -> Optional[torch.Tensor]:
596
return self.language_model.compute_logits(hidden_states,
597
sampling_metadata)
598
vllm/model_executor/models/medusa.py
+11
-5
@@ -65,22 +65,28 @@ class Medusa(nn.Module):
65
def compute_logits(
66
self, hidden_states: List[torch.Tensor],
67
sampling_metadata: SamplingMetadata) -> List[torch.Tensor]:
68
- logits = []
68
+ logits_lst: List[torch.Tensor] = []
69
70
for hs, lm_head in zip(hidden_states, self.lm_heads):
71
_logits = self.logits_processor(lm_head, hs, sampling_metadata)
72
73
+ if _logits is None:
74
+ # _logits should only be None on rank > 0, in which case
75
+ # it should remain true for every lm_head
76
+ assert len(logits_lst) == 0
77
+ continue
78
+
79
if self.token_map is None:
74
- logits.append(_logits)
80
+ logits_lst.append(_logits)
81
else:
76
- logits.append(-torch.inf * torch.ones(
82
+ logits_lst.append(-torch.inf * torch.ones(
83
size=(*_logits.shape[:-1], self.orig_vocab_size),
84
device=_logits.device,
85
dtype=_logits.dtype))
86
81
- logits[-1][..., self.token_map] = _logits
87
+ logits_lst[-1][..., self.token_map] = _logits
88
83
- return logits
89
+ return logits_lst
90
91
def sample(
92
self,
vllm/model_executor/models/minicpm.py
+5
-2
@@ -470,8 +470,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
470
attn_metadata, intermediate_tensors)
471
return hidden_states
472
473
- def compute_logits(self, hidden_states: torch.Tensor,
474
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
473
+ def compute_logits(
474
+ self,
475
+ hidden_states: torch.Tensor,
476
+ sampling_metadata: SamplingMetadata,
477
+ ) -> Optional[torch.Tensor]:
478
hidden_states = hidden_states / self.scale_width
479
if self.config.tie_word_embeddings:
480
lm_head = self.model.embed_tokens
vllm/model_executor/models/minicpmv.py
+5
-2
@@ -630,8 +630,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsVision):
630
)
631
return output
632
633
- def compute_logits(self, hidden_states: torch.Tensor,
634
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
633
+ def compute_logits(
634
+ self,
635
+ hidden_states: torch.Tensor,
636
+ sampling_metadata: SamplingMetadata,
637
+ ) -> Optional[torch.Tensor]:
638
logits = self.logits_processor(self.lm_head, hidden_states,
639
sampling_metadata)
640
return logits
vllm/model_executor/models/mixtral.py
+5
-2
@@ -375,8 +375,11 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
375
attn_metadata, intermediate_tensors)
376
return hidden_states
377
378
- def compute_logits(self, hidden_states: torch.Tensor,
379
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
378
+ def compute_logits(
379
+ self,
380
+ hidden_states: torch.Tensor,
381
+ sampling_metadata: SamplingMetadata,
382
+ ) -> Optional[torch.Tensor]:
383
logits = self.logits_processor(self.lm_head, hidden_states,
384
sampling_metadata)
385
return logits
vllm/model_executor/models/mixtral_quant.py
+5
-2
@@ -362,8 +362,11 @@ class MixtralForCausalLM(nn.Module):
362
attn_metadata)
363
return hidden_states
364
365
- def compute_logits(self, hidden_states: torch.Tensor,
366
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
365
+ def compute_logits(
366
+ self,
367
+ hidden_states: torch.Tensor,
368
+ sampling_metadata: SamplingMetadata,
369
+ ) -> Optional[torch.Tensor]:
370
logits = self.logits_processor(self.lm_head, hidden_states,
371
sampling_metadata)
372
return logits
vllm/model_executor/models/mpt.py
+5
-2
@@ -279,8 +279,11 @@ class MPTForCausalLM(nn.Module):
279
attn_metadata)
280
return hidden_states
281
282
- def compute_logits(self, hidden_states: torch.Tensor,
283
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
282
+ def compute_logits(
283
+ self,
284
+ hidden_states: torch.Tensor,
285
+ sampling_metadata: SamplingMetadata,
286
+ ) -> Optional[torch.Tensor]:
287
logits = self.logits_processor(self.lm_head, hidden_states,
288
sampling_metadata)
289
return logits
vllm/model_executor/models/nemotron.py
+5
-2
@@ -453,8 +453,11 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA):
453
attn_metadata, intermediate_tensors)
454
return model_output
455
456
- def compute_logits(self, hidden_states: torch.Tensor,
457
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
456
+ def compute_logits(
457
+ self,
458
+ hidden_states: torch.Tensor,
459
+ sampling_metadata: SamplingMetadata,
460
+ ) -> Optional[torch.Tensor]:
461
logits = self.logits_processor(self.lm_head, hidden_states,
462
sampling_metadata)
463
return logits
vllm/model_executor/models/olmo.py
+5
-2
@@ -311,8 +311,11 @@ class OlmoForCausalLM(nn.Module):
311
)
312
return hidden_states
313
314
- def compute_logits(self, hidden_states: torch.Tensor,
315
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
314
+ def compute_logits(
315
+ self,
316
+ hidden_states: torch.Tensor,
317
+ sampling_metadata: SamplingMetadata,
318
+ ) -> Optional[torch.Tensor]:
319
logits = self.logits_processor(self.lm_head, hidden_states,
320
sampling_metadata)
321
return logits
vllm/model_executor/models/opt.py
+5
-2
@@ -323,8 +323,11 @@ class OPTForCausalLM(nn.Module):
323
attn_metadata)
324
return hidden_states
325
326
- def compute_logits(self, hidden_states: torch.Tensor,
327
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
326
+ def compute_logits(
327
+ self,
328
+ hidden_states: torch.Tensor,
329
+ sampling_metadata: SamplingMetadata,
330
+ ) -> Optional[torch.Tensor]:
331
logits = self.logits_processor(self.lm_head, hidden_states,
332
sampling_metadata)
333
return logits
vllm/model_executor/models/orion.py
+5
-2
@@ -277,8 +277,11 @@ class OrionForCausalLM(nn.Module):
277
attn_metadata)
278
return hidden_states
279
280
- def compute_logits(self, hidden_states: torch.Tensor,
281
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
280
+ def compute_logits(
281
+ self,
282
+ hidden_states: torch.Tensor,
283
+ sampling_metadata: SamplingMetadata,
284
+ ) -> Optional[torch.Tensor]:
285
logits = self.logits_processor(self.lm_head, hidden_states,
286
sampling_metadata)
287
return logits
vllm/model_executor/models/paligemma.py
+5
-2
@@ -262,8 +262,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
262
return hidden_states
263
264
# Copied from vllm/model_executor/models/gemma.py
265
- def compute_logits(self, hidden_states: torch.Tensor,
266
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
265
+ def compute_logits(
266
+ self,
267
+ hidden_states: torch.Tensor,
268
+ sampling_metadata: SamplingMetadata,
269
+ ) -> Optional[torch.Tensor]:
270
logits = self.logits_processor(self.language_model.embed_tokens,
271
hidden_states, sampling_metadata)
272
return logits
vllm/model_executor/models/persimmon.py
+5
-2
@@ -285,8 +285,11 @@ class PersimmonForCausalLM(nn.Module):
285
)
286
return hidden_states
287
288
- def compute_logits(self, hidden_states: torch.Tensor,
289
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
288
+ def compute_logits(
289
+ self,
290
+ hidden_states: torch.Tensor,
291
+ sampling_metadata: SamplingMetadata,
292
+ ) -> Optional[torch.Tensor]:
293
logits = self.logits_processor(self.lm_head, hidden_states,
294
sampling_metadata)
295
return logits
vllm/model_executor/models/phi.py
+5
-2
@@ -286,8 +286,11 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
286
287
return hidden_states
288
289
- def compute_logits(self, hidden_states: torch.Tensor,
290
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
289
+ def compute_logits(
290
+ self,
291
+ hidden_states: torch.Tensor,
292
+ sampling_metadata: SamplingMetadata,
293
+ ) -> Optional[torch.Tensor]:
294
logits = self.logits_processor(self.lm_head, hidden_states,
295
sampling_metadata, self.lm_head.bias)
296
return logits
vllm/model_executor/models/phi3_small.py
+5
-2
@@ -399,8 +399,11 @@ class Phi3SmallForCausalLM(nn.Module):
399
def get_decoder(self):
400
return self.model
401
402
- def compute_logits(self, hidden_states: torch.Tensor,
403
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
402
+ def compute_logits(
403
+ self,
404
+ hidden_states: torch.Tensor,
405
+ sampling_metadata: SamplingMetadata,
406
+ ) -> Optional[torch.Tensor]:
407
logits = self.logits_processor(self.lm_head, hidden_states,
408
sampling_metadata)
409
if self.dummy_token_indices is not None and logits is not None:
vllm/model_executor/models/phi3v.py
+5
-2
@@ -584,8 +584,11 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
584
585
return hidden_states
586
587
- def compute_logits(self, hidden_states: torch.Tensor,
588
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
587
+ def compute_logits(
588
+ self,
589
+ hidden_states: torch.Tensor,
590
+ sampling_metadata: SamplingMetadata,
591
+ ) -> Optional[torch.Tensor]:
592
logits = self.logits_processor(self.lm_head, hidden_states,
593
sampling_metadata)
594
return logits
vllm/model_executor/models/qwen.py
+5
-2
@@ -281,8 +281,11 @@ class QWenLMHeadModel(nn.Module):
281
device=device),
282
})
283
284
- def compute_logits(self, hidden_states: torch.Tensor,
285
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
284
+ def compute_logits(
285
+ self,
286
+ hidden_states: torch.Tensor,
287
+ sampling_metadata: SamplingMetadata,
288
+ ) -> Optional[torch.Tensor]:
289
logits = self.logits_processor(self.lm_head, hidden_states,
290
sampling_metadata)
291
return logits
vllm/model_executor/models/qwen2.py
+5
-2
@@ -362,8 +362,11 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
362
attn_metadata, intermediate_tensors)
363
return hidden_states
364
365
- def compute_logits(self, hidden_states: torch.Tensor,
366
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
365
+ def compute_logits(
366
+ self,
367
+ hidden_states: torch.Tensor,
368
+ sampling_metadata: SamplingMetadata,
369
+ ) -> Optional[torch.Tensor]:
370
logits = self.logits_processor(self.lm_head, hidden_states,
371
sampling_metadata)
372
return logits
vllm/model_executor/models/qwen2_moe.py
+5
-2
@@ -400,8 +400,11 @@ class Qwen2MoeForCausalLM(nn.Module):
400
attn_metadata, intermediate_tensors)
401
return hidden_states
402
403
- def compute_logits(self, hidden_states: torch.Tensor,
404
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
403
+ def compute_logits(
404
+ self,
405
+ hidden_states: torch.Tensor,
406
+ sampling_metadata: SamplingMetadata,
407
+ ) -> Optional[torch.Tensor]:
408
logits = self.logits_processor(self.lm_head, hidden_states,
409
sampling_metadata)
410
return logits
vllm/model_executor/models/stablelm.py
+5
-2
@@ -258,8 +258,11 @@ class StablelmForCausalLM(nn.Module):
258
attn_metadata)
259
return hidden_states
260
261
- def compute_logits(self, hidden_states: torch.Tensor,
262
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
261
+ def compute_logits(
262
+ self,
263
+ hidden_states: torch.Tensor,
264
+ sampling_metadata: SamplingMetadata,
265
+ ) -> Optional[torch.Tensor]:
266
logits = self.logits_processor(self.lm_head, hidden_states,
267
sampling_metadata)
268
return logits
vllm/model_executor/models/starcoder2.py
+5
-2
@@ -268,8 +268,11 @@ class Starcoder2ForCausalLM(nn.Module):
268
attn_metadata)
269
return hidden_states
270
271
- def compute_logits(self, hidden_states: torch.Tensor,
272
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
271
+ def compute_logits(
272
+ self,
273
+ hidden_states: torch.Tensor,
274
+ sampling_metadata: SamplingMetadata,
275
+ ) -> Optional[torch.Tensor]:
276
logits = self.logits_processor(self.lm_head, hidden_states,
277
sampling_metadata)
278
return logits
vllm/model_executor/models/xverse.py
+5
-2
@@ -328,8 +328,11 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
328
attn_metadata)
329
return hidden_states
330
331
- def compute_logits(self, hidden_states: torch.Tensor,
332
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
331
+ def compute_logits(
332
+ self,
333
+ hidden_states: torch.Tensor,
334
+ sampling_metadata: SamplingMetadata,
335
+ ) -> Optional[torch.Tensor]:
336
logits = self.logits_processor(self.lm_head, hidden_states,
337
sampling_metadata)
338
return logits
vllm/outputs.py
+5
-3
@@ -1,6 +1,8 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import List, Optional, Tuple, Union
3
+from typing import List, Optional
4
+from typing import Sequence as GenericSequence
5
+from typing import Union
6
7
from vllm.lora.request import LoRARequest
8
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -28,7 +30,7 @@ class CompletionOutput:
30
31
index: int
32
text: str
31
- token_ids: Tuple[int, ...]
33
+ token_ids: GenericSequence[int]
34
cumulative_logprob: Optional[float]
35
logprobs: Optional[SampleLogprobs]
36
finish_reason: Optional[str] = None
@@ -139,7 +141,7 @@ class RequestOutput:
141
CompletionOutput(
142
seqs.index(seq),
143
seq.get_output_text_to_return(text_buffer_length),
142
- seq.data._output_token_ids, # type: ignore
144
+ seq.data._output_token_ids,
145
seq.get_cumulative_logprob() if include_logprobs else None,
146
seq.output_logprobs if include_logprobs else None,
147
SequenceStatus.get_finished_reason(seq.status),