@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7025b11d94
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix weight loading for Chameleon when TP>1 (#7410)

Cyrus Leung committed Aug 13, 2024 at 13:33 UTC 7025b11d949b4efeb2584690c35f919c77027368
59 files changed +414 -205
tests/conftest.py
+18 -8
@@ -4,7 +4,8 @@ import os
4 import sys
5 from collections import UserList
6 from enum import Enum
7 -from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union
7 +from typing import (Any, Callable, Dict, List, Optional, Tuple, TypedDict,
8 + TypeVar, Union)
9
10 import pytest
11 import torch
@@ -27,7 +28,7 @@ from vllm.logger import init_logger
28 from vllm.outputs import RequestOutput
29 from vllm.sequence import SampleLogprobs
30 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
30 - is_cpu)
31 + identity, is_cpu)
32
33 logger = init_logger(__name__)
34
@@ -197,6 +198,8 @@ class HfRunner:
198 is_embedding_model: bool = False,
199 is_vision_model: bool = False,
200 is_encoder_decoder_model: bool = False,
201 + postprocess_inputs: Callable[[BatchEncoding],
202 + BatchEncoding] = identity,
203 ) -> None:
204 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
205
@@ -242,12 +245,14 @@ class HfRunner:
245 torch_dtype=torch_dtype,
246 trust_remote_code=True,
247 )
245 - except Exception:
248 + except Exception as exc:
249 logger.warning(
247 - "Unable to auto-load processor from HuggingFace for "
248 - "model %s. Using tokenizer instead.", model_name)
250 + "Unable to auto-load HuggingFace processor for model (%s). "
251 + "Using tokenizer instead. Reason: %s", model_name, exc)
252 self.processor = self.tokenizer
253
254 + self.postprocess_inputs = postprocess_inputs
255 +
256 def generate(
257 self,
258 prompts: List[str],
@@ -267,6 +272,7 @@ class HfRunner:
272 processor_kwargs["images"] = images[i]
273
274 inputs = self.processor(**processor_kwargs)
275 + inputs = self.postprocess_inputs(inputs)
276
277 output_ids = self.model.generate(
278 **self.wrap_device(inputs),
@@ -336,6 +342,7 @@ class HfRunner:
342 processor_kwargs["images"] = images[i]
343
344 inputs = self.processor(**processor_kwargs)
345 + inputs = self.postprocess_inputs(inputs)
346
347 output = self.model.generate(
348 **self.wrap_device(inputs),
@@ -420,6 +427,7 @@ class HfRunner:
427 processor_kwargs["images"] = images[i]
428
429 inputs = self.processor(**processor_kwargs)
430 + inputs = self.postprocess_inputs(inputs)
431
432 output = self.model.generate(
433 **self.wrap_device(inputs),
@@ -552,7 +560,8 @@ class VllmRunner:
560 self,
561 prompts: List[str],
562 sampling_params: SamplingParams,
555 - images: Optional[List[Image.Image]] = None,
563 + images: Optional[Union[List[Image.Image],
564 + List[List[Image.Image]]]] = None,
565 ) -> List[Tuple[List[List[int]], List[str]]]:
566 if images is not None:
567 assert len(prompts) == len(images)
@@ -587,7 +596,7 @@ class VllmRunner:
596 for req_output in req_outputs:
597 for sample in req_output.outputs:
598 output_str = sample.text
590 - output_ids = sample.token_ids
599 + output_ids = list(sample.token_ids)
600 output_logprobs = sample.logprobs
601 outputs.append((output_ids, output_str, output_logprobs))
602 return outputs
@@ -596,7 +605,8 @@ class VllmRunner:
605 self,
606 prompts: List[str],
607 sampling_params: SamplingParams,
599 - images: Optional[List[Image.Image]] = None,
608 + images: Optional[Union[List[Image.Image],
609 + List[List[Image.Image]]]] = None,
610 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
611 assert sampling_params.logprobs is not None
612
tests/distributed/test_multimodal_broadcast.py
+4
@@ -18,8 +18,10 @@ from ..utils import fork_new_process_for_each_test
18 @pytest.mark.parametrize("model, distributed_executor_backend", [
19 ("llava-hf/llava-1.5-7b-hf", "ray"),
20 ("llava-hf/llava-v1.6-mistral-7b-hf", "ray"),
21 + ("facebook/chameleon-7b", "ray"),
22 ("llava-hf/llava-1.5-7b-hf", "mp"),
23 ("llava-hf/llava-v1.6-mistral-7b-hf", "mp"),
24 + ("facebook/chameleon-7b", "mp"),
25 ])
26 @fork_new_process_for_each_test
27 def test_models(hf_runner, vllm_runner, image_assets, model: str,
@@ -34,6 +36,8 @@ def test_models(hf_runner, vllm_runner, image_assets, model: str,
36 from ..models.test_llava import models, run_test
37 elif model.startswith("llava-hf/llava-v1.6"):
38 from ..models.test_llava_next import models, run_test
39 + elif model.startswith("facebook/chameleon"):
40 + from ..models.test_chameleon import models, run_test
41 else:
42 raise NotImplementedError(f"Unsupported model: {model}")
43
tests/entrypoints/openai/test_oot_registration.py
+6 -2
@@ -1,5 +1,6 @@
1 import sys
2 import time
3 +from typing import Optional
4
5 import torch
6 from openai import OpenAI, OpenAIError
@@ -17,8 +18,11 @@ assert chatml_jinja_path.exists()
18
19 class MyOPTForCausalLM(OPTForCausalLM):
20
20 - def compute_logits(self, hidden_states: torch.Tensor,
21 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
21 + def compute_logits(
22 + self,
23 + hidden_states: torch.Tensor,
24 + sampling_metadata: SamplingMetadata,
25 + ) -> Optional[torch.Tensor]:
26 # this dummy model always predicts the first token
27 logits = super().compute_logits(hidden_states, sampling_metadata)
28 logits.zero_()
tests/models/test_chameleon.py
+58 -33
@@ -1,11 +1,13 @@
1 -import re
1 from typing import List, Optional, Type
2
3 import pytest
4 +from transformers import BatchEncoding
5
6 from vllm.multimodal.utils import rescale_image_size
7 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
8
8 -from ..conftest import IMAGE_ASSETS, VllmRunner, _ImageAssets
9 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10 +from .utils import check_outputs_equal
11
12 pytestmark = pytest.mark.vlm
13
@@ -19,9 +21,8 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21 models = ["facebook/chameleon-7b"]
22
23
22 -#TODO (ywang96): Add correctness test when chameleon is
23 -# available on transformers.
24 def run_test(
25 + hf_runner: Type[HfRunner],
26 vllm_runner: Type[VllmRunner],
27 image_assets: _ImageAssets,
28 model: str,
@@ -29,13 +30,20 @@ def run_test(
30 size_factors: List[float],
31 dtype: str,
32 max_tokens: int,
33 + num_logprobs: int,
34 tensor_parallel_size: int,
35 distributed_executor_backend: Optional[str] = None,
36 ):
35 - """Test if the model can generate text given
36 - a batch of images and prompts.
37 -
37 + """Inference result should be the same between hf and vllm.
38 +
39 + All the image fixtures for the test is under tests/images.
40 + For huggingface runner, we provide the PIL images as input.
41 + For vllm runner, we provide MultiModalDataDict objects
42 + and corresponding vision language config as input.
43 + Note, the text input is also adjusted to abide by vllm contract.
44 + The text output is sanitized to be able to compare with hf.
45 """
46 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
47 images = [asset.pil_image for asset in image_assets]
48
49 inputs_per_image = [(
@@ -50,35 +58,49 @@ def run_test(
58 distributed_executor_backend=distributed_executor_backend,
59 enforce_eager=True) as vllm_model:
60
53 - for prompts, images in inputs_per_image:
54 - vllm_outputs = vllm_model.generate_greedy(prompts,
55 - max_tokens,
56 - images=images)
57 - for i in range(len(vllm_outputs)):
58 -
59 - # format prompt back to original
60 - replacements = {
61 - "<racm3:break>": "",
62 - "<eoss>": "",
63 - "<reserved08706>": ""
64 - }
65 - pattern = '|'.join(replacements.keys())
66 - vllm_result = re.sub(
67 - pattern,
68 - lambda match: replacements[match.group(0)], #noqa B023
69 - vllm_outputs[i][1])
70 - vllm_result = vllm_result.replace("<image>", "", 1023)
71 - assert vllm_result[:len(prompts[i])] == prompts[i]
72 -
73 - # assert at least 10 new characters are generated
74 - # (to take stop token into account)
75 - assert len(vllm_outputs[i][1]) - len(prompts[i]) > 10
61 + vllm_outputs_per_image = [
62 + vllm_model.generate_greedy_logprobs(prompts,
63 + max_tokens,
64 + num_logprobs=num_logprobs,
65 + images=images)
66 + for prompts, images in inputs_per_image
67 + ]
68 +
69 + def process(hf_inputs: BatchEncoding):
70 + hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
71 + .to(torch_dtype) # type: ignore
72 + return hf_inputs
73 +
74 + with hf_runner(model,
75 + dtype=dtype,
76 + postprocess_inputs=process,
77 + is_vision_model=True) as hf_model:
78 + hf_outputs_per_image = [
79 + hf_model.generate_greedy_logprobs_limit(prompts,
80 + max_tokens,
81 + num_logprobs=num_logprobs,
82 + images=images)
83 + for prompts, images in inputs_per_image
84 + ]
85 +
86 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
87 + vllm_outputs_per_image):
88 + # HF Logprobs include image tokens, unlike vLLM, so we don't directly
89 + # compare them
90 + check_outputs_equal(
91 + outputs_0_lst=[outputs[:2] for outputs in hf_outputs],
92 + outputs_1_lst=[outputs[:2] for outputs in vllm_outputs],
93 + name_0="hf",
94 + name_1="vllm",
95 + )
96
97
98 @pytest.mark.parametrize("model", models)
99 @pytest.mark.parametrize(
100 "size_factors",
101 [
102 + # No image
103 + [],
104 # Single-scale
105 [1.0],
106 # Single-scale, batched
@@ -88,15 +110,18 @@ def run_test(
110 ],
111 )
112 @pytest.mark.parametrize("dtype", ["bfloat16"])
91 -@pytest.mark.parametrize("max_tokens", [128])
92 -def test_models(vllm_runner, image_assets, model, size_factors, dtype: str,
93 - max_tokens: int) -> None:
113 +@pytest.mark.parametrize("max_tokens", [8])
114 +@pytest.mark.parametrize("num_logprobs", [5])
115 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
116 + dtype, max_tokens, num_logprobs) -> None:
117 run_test(
118 + hf_runner,
119 vllm_runner,
120 image_assets,
121 model,
122 size_factors=size_factors,
123 dtype=dtype,
124 max_tokens=max_tokens,
125 + num_logprobs=num_logprobs,
126 tensor_parallel_size=1,
127 )
tests/models/test_llava.py
+13 -8
@@ -1,7 +1,7 @@
1 from typing import List, Optional, Tuple, Type
2
3 import pytest
4 -from transformers import AutoConfig, AutoTokenizer
4 +from transformers import AutoConfig, AutoTokenizer, BatchEncoding
5
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
@@ -110,16 +110,21 @@ def run_test(
110 for prompts, images in inputs_per_image
111 ]
112
113 - with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
114 - if mantis_processor is not None:
113 + if mantis_processor is not None:
114
116 - def process(*args, **kwargs):
117 - output = mantis_processor(*args, **kwargs)
118 - output["pixel_values"] = output["pixel_values"].to(torch_dtype)
119 - return output
115 + def process(hf_inputs: BatchEncoding):
116 + hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
117 + .to(torch_dtype) # type: ignore
118 + return hf_inputs
119 + else:
120
121 - hf_model.processor = process
121 + def process(hf_inputs: BatchEncoding):
122 + return hf_inputs
123
124 + with hf_runner(model,
125 + dtype=dtype,
126 + postprocess_inputs=process,
127 + is_vision_model=True) as hf_model:
128 hf_outputs_per_image = [
129 hf_model.generate_greedy_logprobs_limit(prompts,
130 max_tokens,
tests/models/test_minicpmv.py
+9 -24
@@ -1,10 +1,9 @@
1 -from collections import UserDict
1 from typing import List, Optional, Tuple, Type
2
3 import pytest
4 import torch
5 import torch.types
7 -from transformers import BatchFeature
6 +from transformers import BatchEncoding
7
8 from vllm.multimodal.utils import rescale_image_size
9 from vllm.sequence import SampleLogprobs
@@ -14,18 +13,6 @@ from .utils import check_logprobs_close
13
14 pytestmark = pytest.mark.vlm
15
17 -
18 -class NestedInputs(UserDict):
19 -
20 - def __init__(self, model_inputs: BatchFeature):
21 - super().__init__({"model_inputs": model_inputs})
22 -
23 - self.model_inputs = model_inputs
24 -
25 - def to(self, device: torch.types.Device):
26 - return NestedInputs(self.model_inputs.to(device))
27 -
28 -
16 # The image token is placed before "user" on purpose so that the test can pass
17 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18 "stop_sign":
@@ -41,6 +28,10 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
28 models = ["openbmb/MiniCPM-Llama3-V-2_5"]
29
30
31 +def _wrap_inputs(hf_inputs: BatchEncoding) -> BatchEncoding:
32 + return BatchEncoding({"model_inputs": hf_inputs})
33 +
34 +
35 def trunc_hf_output(hf_output: Tuple[List[int], str,
36 Optional[SampleLogprobs]]):
37 output_ids, output_str, out_logprobs = hf_output
@@ -105,11 +96,8 @@ def run_test(
96 for prompts, images in inputs_per_image
97 ]
98
108 - with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
109 - hf_processor = hf_model.processor
110 - hf_model.processor = lambda **kw: NestedInputs(
111 - hf_processor(**kw) # type: ignore
112 - )
99 + hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
100 + with hf_model, torch.no_grad():
101 hf_outputs_per_image = [
102 hf_model.generate_greedy_logprobs_limit(prompts,
103 max_tokens,
@@ -224,11 +212,8 @@ def run_multi_image_test(
212 for prompts, images in inputs_per_case
213 ]
214
227 - with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
228 - hf_processor = hf_model.processor
229 - hf_model.processor = lambda **kw: NestedInputs(
230 - hf_processor(**kw) # type: ignore
231 - )
215 + hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
216 + with hf_model, torch.no_grad():
217 hf_outputs_per_case = [
218 hf_model.generate_greedy_logprobs_limit(prompts,
219 max_tokens,
tests/models/test_oot_registration.py
+7 -2
@@ -1,3 +1,5 @@
1 +from typing import Optional
2 +
3 import torch
4
5 from vllm import LLM, ModelRegistry, SamplingParams
@@ -7,8 +9,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
9
10 class MyOPTForCausalLM(OPTForCausalLM):
11
10 - def compute_logits(self, hidden_states: torch.Tensor,
11 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
12 + def compute_logits(
13 + self,
14 + hidden_states: torch.Tensor,
15 + sampling_metadata: SamplingMetadata,
16 + ) -> Optional[torch.Tensor]:
17 # this dummy model always predicts the first token
18 logits = super().compute_logits(hidden_states, sampling_metadata)
19 logits.zero_()
vllm/distributed/communication_op.py
+1 -1
@@ -19,7 +19,7 @@ def tensor_model_parallel_all_gather(input_: torch.Tensor,
19
20 def tensor_model_parallel_gather(input_: torch.Tensor,
21 dst: int = 0,
22 - dim: int = -1) -> torch.Tensor:
22 + dim: int = -1) -> Optional[torch.Tensor]:
23 """Gather the input tensor across model parallel group."""
24 return get_tp_group().gather(input_, dst, dim)
25
vllm/distributed/parallel_state.py
+1 -1
@@ -329,7 +329,7 @@ class GroupCoordinator:
329 def gather(self,
330 input_: torch.Tensor,
331 dst: int = 0,
332 - dim: int = -1) -> torch.Tensor:
332 + dim: int = -1) -> Optional[torch.Tensor]:
333 """
334 NOTE: We assume that the input tensor is on the same device across
335 all the ranks.
vllm/model_executor/layers/logits_processor.py
+8 -4
@@ -50,7 +50,7 @@ class LogitsProcessor(nn.Module):
50 hidden_states: torch.Tensor,
51 sampling_metadata: SamplingMetadata,
52 embedding_bias: Optional[torch.Tensor] = None,
53 - ) -> torch.Tensor:
53 + ) -> Optional[torch.Tensor]:
54 if self.logits_as_input:
55 logits = hidden_states
56 else:
@@ -73,14 +73,18 @@ class LogitsProcessor(nn.Module):
73
74 return logits
75
76 - def _get_logits(self, hidden_states: torch.Tensor,
77 - lm_head: VocabParallelEmbedding,
78 - embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
76 + def _get_logits(
77 + self,
78 + hidden_states: torch.Tensor,
79 + lm_head: VocabParallelEmbedding,
80 + embedding_bias: Optional[torch.Tensor],
81 + ) -> Optional[torch.Tensor]:
82 # Get the logits for the next tokens.
83 logits = lm_head.linear_method.apply(lm_head,
84 hidden_states,
85 bias=embedding_bias)
86 if self.use_gather:
87 + # None may be returned for rank > 0
88 logits = tensor_model_parallel_gather(logits)
89 else:
90 # Gather is not supported for some devices such as TPUs.
vllm/model_executor/model_loader/weight_utils.py
+25 -2
@@ -19,6 +19,7 @@ from tqdm.auto import tqdm
19 from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
20
21 from vllm.config import LoadConfig, ModelConfig
22 +from vllm.distributed import get_tensor_model_parallel_rank
23 from vllm.logger import init_logger
24 from vllm.model_executor.layers.quantization import (QuantizationConfig,
25 get_quantization_config)
@@ -514,8 +515,30 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
515 def default_weight_loader(param: torch.Tensor,
516 loaded_weight: torch.Tensor) -> None:
517 """Default weight loader."""
517 - assert param.size() == loaded_weight.size()
518 - param.data.copy_(loaded_weight)
518 + try:
519 + assert param.size() == loaded_weight.size(), (
520 + f"Attempted to load weight ({loaded_weight.size()}) "
521 + f"into parameter ({param.size()})")
522 +
523 + param.data.copy_(loaded_weight)
524 + except Exception:
525 + # NOTE: This exception is added for the purpose of setting breakpoint to
526 + # debug weight loading issues.
527 + raise
528 +
529 +
530 +def row_parallel_weight_loader(param: torch.Tensor,
531 + loaded_weight: torch.Tensor) -> None:
532 + """Load weights that are row-parallelized."""
533 + tp_rank = get_tensor_model_parallel_rank()
534 + shard_dim = 0 if param.dim() != 1 else None
535 +
536 + if shard_dim is not None:
537 + shard_size = param.data.shape[shard_dim]
538 + start_idx = tp_rank * shard_size
539 + loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size)
540 +
541 + return default_weight_loader(param, loaded_weight)
542
543
544 def initialize_dummy_weights(
vllm/model_executor/models/arctic.py
+5 -2
@@ -433,8 +433,11 @@ class ArcticForCausalLM(nn.Module):
433 attn_metadata)
434 return hidden_states
435
436 - def compute_logits(self, hidden_states: torch.Tensor,
437 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
436 + def compute_logits(
437 + self,
438 + hidden_states: torch.Tensor,
439 + sampling_metadata: SamplingMetadata,
440 + ) -> Optional[torch.Tensor]:
441 logits = self.logits_processor(self.lm_head, hidden_states,
442 sampling_metadata)
443 return logits
vllm/model_executor/models/baichuan.py
+5 -2
@@ -346,8 +346,11 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
346 attn_metadata)
347 return hidden_states
348
349 - def compute_logits(self, hidden_states: torch.Tensor,
350 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
349 + def compute_logits(
350 + self,
351 + hidden_states: torch.Tensor,
352 + sampling_metadata: SamplingMetadata,
353 + ) -> Optional[torch.Tensor]:
354 logits = self.logits_processor(self.lm_head, hidden_states,
355 sampling_metadata)
356 return logits
vllm/model_executor/models/bart.py
+5 -2
@@ -872,8 +872,11 @@ class BartForConditionalGeneration(nn.Module):
872 return self.model(input_ids, positions, encoder_input_ids,
873 encoder_positions, kv_caches, attn_metadata)
874
875 - def compute_logits(self, hidden_states: torch.Tensor,
876 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
875 + def compute_logits(
876 + self,
877 + hidden_states: torch.Tensor,
878 + sampling_metadata: SamplingMetadata,
879 + ) -> Optional[torch.Tensor]:
880 logits = self.logits_processor(self.lm_head, hidden_states,
881 sampling_metadata)
882 return logits
vllm/model_executor/models/blip2.py
+5 -2
@@ -637,8 +637,11 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsVision):
637
638 return hidden_states
639
640 - def compute_logits(self, hidden_states: torch.Tensor,
641 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
640 + def compute_logits(
641 + self,
642 + hidden_states: torch.Tensor,
643 + sampling_metadata: SamplingMetadata,
644 + ) -> Optional[torch.Tensor]:
645 logits = self.logits_processor(self.get_lm_head(), hidden_states,
646 sampling_metadata)
647 return logits
vllm/model_executor/models/bloom.py
+5 -2
@@ -292,8 +292,11 @@ class BloomForCausalLM(nn.Module):
292 attn_metadata)
293 return hidden_states
294
295 - def compute_logits(self, hidden_states: torch.Tensor,
296 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
295 + def compute_logits(
296 + self,
297 + hidden_states: torch.Tensor,
298 + sampling_metadata: SamplingMetadata,
299 + ) -> Optional[torch.Tensor]:
300 logits = self.logits_processor(self.lm_head, hidden_states,
301 sampling_metadata)
302 return logits
vllm/model_executor/models/chameleon.py
+18 -5
@@ -25,8 +25,10 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
25 from vllm.model_executor.layers.sampler import Sampler
26 from vllm.model_executor.layers.vocab_parallel_embedding import (
27 ParallelLMHead, VocabParallelEmbedding)
28 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28 +from vllm.model_executor.model_loader.weight_utils import (
29 + default_weight_loader, row_parallel_weight_loader)
30 from vllm.model_executor.sampling_metadata import SamplingMetadata
31 +from vllm.model_executor.utils import set_weight_attrs
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.image import (cached_get_tokenizer,
34 repeat_and_pad_image_tokens)
@@ -141,6 +143,11 @@ class ChameleonLayerNorm(nn.LayerNorm):
143 super().__init__(hidden_size, *args, **kwargs)
144 self.normalized_shape = (hidden_size[-1], )
145
146 + set_weight_attrs(self.weight,
147 + {"weight_loader": row_parallel_weight_loader})
148 + set_weight_attrs(self.bias,
149 + {"weight_loader": row_parallel_weight_loader})
150 +
151 def forward(self, hidden_states):
152 hidden_states = F.layer_norm(hidden_states,
153 self.normalized_shape,
@@ -697,6 +704,8 @@ class ChameleonVQVAEEncoder(nn.Module):
704 )
705
706 def forward(self, pixel_values: torch.Tensor):
707 + pixel_values = pixel_values.to(self.conv_in.weight.dtype)
708 +
709 # downsampling
710 hidden_states = [self.conv_in(pixel_values)]
711 for i_level in range(self.num_resolutions):
@@ -959,15 +968,19 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
968 attn_metadata)
969 return hidden_states
970
962 - def compute_logits(self, hidden_states: torch.Tensor,
963 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
971 + def compute_logits(
972 + self,
973 + hidden_states: torch.Tensor,
974 + sampling_metadata: SamplingMetadata,
975 + ) -> Optional[torch.Tensor]:
976 logits = self.logits_processor(self.lm_head, hidden_states,
977 sampling_metadata)
978
979 # Disallow image tokens which does not include special
980 # begin-image and end-image tokens
969 - image_tokens = self.model.vocabulary_mapping.image_tokens
970 - logits[:, image_tokens] = torch.finfo(logits.dtype).min
981 + if logits is not None:
982 + image_tokens = self.model.vocabulary_mapping.image_tokens
983 + logits[:, image_tokens] = torch.finfo(logits.dtype).min
984
985 return logits
986
vllm/model_executor/models/chatglm.py
+5 -2
@@ -372,8 +372,11 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
372 attn_metadata)
373 return hidden_states
374
375 - def compute_logits(self, hidden_states: torch.Tensor,
376 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
375 + def compute_logits(
376 + self,
377 + hidden_states: torch.Tensor,
378 + sampling_metadata: SamplingMetadata,
379 + ) -> Optional[torch.Tensor]:
380 logits = self.logits_processor(self.lm_head, hidden_states,
381 sampling_metadata)
382 return logits
vllm/model_executor/models/commandr.py
+10 -19
@@ -25,13 +25,11 @@ from typing import Iterable, List, Optional, Set, Tuple
25 import torch
26 import torch.utils.checkpoint
27 from torch import nn
28 -from torch.nn.parameter import Parameter
28 from transformers import CohereConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import CacheConfig, LoRAConfig
33 -from vllm.distributed import (get_tensor_model_parallel_rank,
34 - get_tensor_model_parallel_world_size)
32 +from vllm.distributed import get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
35 QKVParallelLinear,
@@ -43,7 +41,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 VocabParallelEmbedding)
46 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44 +from vllm.model_executor.model_loader.weight_utils import (
45 + default_weight_loader, row_parallel_weight_loader)
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.model_executor.utils import set_weight_attrs
48 from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -67,25 +66,14 @@ class LayerNorm(nn.Module):
66 super().__init__()
67 self.weight = nn.Parameter(torch.ones(param_shape))
68 self.variance_epsilon = eps
70 - set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
69 + set_weight_attrs(self.weight,
70 + {"weight_loader": row_parallel_weight_loader})
71
72 def forward(self, hidden_states, residuals=None):
73 hidden_states = layer_norm_func(hidden_states, self.weight,
74 self.variance_epsilon)
75 return hidden_states, residuals
76
77 - def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
78 - tp_rank = get_tensor_model_parallel_rank()
79 - shard_dim = 0 if param.dim() != 1 else None
80 - param_data = param.data
81 - if shard_dim is not None:
82 - shard_size = param_data.shape[shard_dim]
83 - start_idx = tp_rank * shard_size
84 - loaded_weight = loaded_weight.narrow(shard_dim, start_idx,
85 - shard_size)
86 - assert param_data.shape == loaded_weight.shape
87 - param_data.copy_(loaded_weight)
88 -
77
78 # Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
79 class CohereMLP(nn.Module):
@@ -359,8 +347,11 @@ class CohereForCausalLM(nn.Module):
347 attn_metadata)
348 return hidden_states
349
362 - def compute_logits(self, hidden_states: torch.Tensor,
363 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
350 + def compute_logits(
351 + self,
352 + hidden_states: torch.Tensor,
353 + sampling_metadata: SamplingMetadata,
354 + ) -> Optional[torch.Tensor]:
355 is_not_lora = hasattr(self.model.embed_tokens, 'weight')
356 if is_not_lora:
357 logits = self.logits_processor(self.model.embed_tokens,
vllm/model_executor/models/dbrx.py
+5 -2
@@ -388,8 +388,11 @@ class DbrxForCausalLM(nn.Module):
388 attn_metadata)
389 return hidden_states
390
391 - def compute_logits(self, hidden_states: torch.Tensor,
392 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
391 + def compute_logits(
392 + self,
393 + hidden_states: torch.Tensor,
394 + sampling_metadata: SamplingMetadata,
395 + ) -> Optional[torch.Tensor]:
396 logits = self.logits_processor(self.lm_head, hidden_states,
397 sampling_metadata)
398 return logits
vllm/model_executor/models/deepseek.py
+5 -2
@@ -395,8 +395,11 @@ class DeepseekForCausalLM(nn.Module):
395 attn_metadata)
396 return hidden_states
397
398 - def compute_logits(self, hidden_states: torch.Tensor,
399 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
398 + def compute_logits(
399 + self,
400 + hidden_states: torch.Tensor,
401 + sampling_metadata: SamplingMetadata,
402 + ) -> Optional[torch.Tensor]:
403 logits = self.logits_processor(self.lm_head, hidden_states,
404 sampling_metadata)
405 return logits
vllm/model_executor/models/deepseek_v2.py
+5 -2
@@ -505,8 +505,11 @@ class DeepseekV2ForCausalLM(nn.Module):
505 attn_metadata, intermediate_tensors)
506 return hidden_states
507
508 - def compute_logits(self, hidden_states: torch.Tensor,
509 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
508 + def compute_logits(
509 + self,
510 + hidden_states: torch.Tensor,
511 + sampling_metadata: SamplingMetadata,
512 + ) -> Optional[torch.Tensor]:
513 logits = self.logits_processor(self.lm_head, hidden_states,
514 sampling_metadata)
515 return logits
vllm/model_executor/models/falcon.py
+5 -2
@@ -420,8 +420,11 @@ class FalconForCausalLM(nn.Module):
420 )
421 return hidden_states
422
423 - def compute_logits(self, hidden_states: torch.Tensor,
424 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
423 + def compute_logits(
424 + self,
425 + hidden_states: torch.Tensor,
426 + sampling_metadata: SamplingMetadata,
427 + ) -> Optional[torch.Tensor]:
428 logits = self.logits_processor(self.lm_head, hidden_states,
429 sampling_metadata)
430 return logits
vllm/model_executor/models/fuyu.py
+5 -2
@@ -287,8 +287,11 @@ class FuyuForCausalLM(nn.Module, SupportsVision):
287 )
288 return hidden_states
289
290 - def compute_logits(self, hidden_states: torch.Tensor,
291 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
290 + def compute_logits(
291 + self,
292 + hidden_states: torch.Tensor,
293 + sampling_metadata: SamplingMetadata,
294 + ) -> Optional[torch.Tensor]:
295 logits = self.language_model.logits_processor(
296 self.language_model.lm_head, hidden_states, sampling_metadata)
297 return logits
vllm/model_executor/models/gemma.py
+5 -2
@@ -352,8 +352,11 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
352 attn_metadata)
353 return hidden_states
354
355 - def compute_logits(self, hidden_states: torch.Tensor,
356 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
355 + def compute_logits(
356 + self,
357 + hidden_states: torch.Tensor,
358 + sampling_metadata: SamplingMetadata,
359 + ) -> Optional[torch.Tensor]:
360 logits = self.logits_processor(self.model.embed_tokens, hidden_states,
361 sampling_metadata)
362 return logits
vllm/model_executor/models/gemma2.py
+5 -2
@@ -343,8 +343,11 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
343 attn_metadata)
344 return hidden_states
345
346 - def compute_logits(self, hidden_states: torch.Tensor,
347 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
346 + def compute_logits(
347 + self,
348 + hidden_states: torch.Tensor,
349 + sampling_metadata: SamplingMetadata,
350 + ) -> Optional[torch.Tensor]:
351 logits = self.logits_processor(self.model.embed_tokens, hidden_states,
352 sampling_metadata)
353 return logits
vllm/model_executor/models/gpt2.py
+5 -2
@@ -265,8 +265,11 @@ class GPT2LMHeadModel(nn.Module):
265 attn_metadata, intermediate_tensors)
266 return hidden_states
267
268 - def compute_logits(self, hidden_states: torch.Tensor,
269 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
268 + def compute_logits(
269 + self,
270 + hidden_states: torch.Tensor,
271 + sampling_metadata: SamplingMetadata,
272 + ) -> Optional[torch.Tensor]:
273 logits = self.logits_processor(self.lm_head, hidden_states,
274 sampling_metadata)
275 return logits
vllm/model_executor/models/gpt_bigcode.py
+5 -2
@@ -279,8 +279,11 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
279 attn_metadata)
280 return hidden_states
281
282 - def compute_logits(self, hidden_states: torch.Tensor,
283 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
282 + def compute_logits(
283 + self,
284 + hidden_states: torch.Tensor,
285 + sampling_metadata: SamplingMetadata,
286 + ) -> Optional[torch.Tensor]:
287 logits = self.logits_processor(self.lm_head, hidden_states,
288 sampling_metadata)
289 return logits
vllm/model_executor/models/gpt_j.py
+5 -2
@@ -246,8 +246,11 @@ class GPTJForCausalLM(nn.Module):
246 attn_metadata)
247 return hidden_states
248
249 - def compute_logits(self, hidden_states: torch.Tensor,
250 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
249 + def compute_logits(
250 + self,
251 + hidden_states: torch.Tensor,
252 + sampling_metadata: SamplingMetadata,
253 + ) -> Optional[torch.Tensor]:
254 logits = self.logits_processor(self.lm_head, hidden_states,
255 sampling_metadata, self.lm_head.bias)
256 return logits
vllm/model_executor/models/gpt_neox.py
+5 -2
@@ -258,8 +258,11 @@ class GPTNeoXForCausalLM(nn.Module):
258 attn_metadata)
259 return hidden_states
260
261 - def compute_logits(self, hidden_states: torch.Tensor,
262 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
261 + def compute_logits(
262 + self,
263 + hidden_states: torch.Tensor,
264 + sampling_metadata: SamplingMetadata,
265 + ) -> Optional[torch.Tensor]:
266 logits = self.logits_processor(self.embed_out, hidden_states,
267 sampling_metadata)
268 return logits
vllm/model_executor/models/internlm2.py
+5 -2
@@ -279,8 +279,11 @@ class InternLM2ForCausalLM(nn.Module):
279 attn_metadata)
280 return hidden_states
281
282 - def compute_logits(self, hidden_states: torch.Tensor,
283 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
282 + def compute_logits(
283 + self,
284 + hidden_states: torch.Tensor,
285 + sampling_metadata: SamplingMetadata,
286 + ) -> Optional[torch.Tensor]:
287 logits = self.logits_processor(self.output, hidden_states,
288 sampling_metadata)
289 return logits
vllm/model_executor/models/internvl.py
+5 -2
@@ -466,8 +466,11 @@ class InternVLChatModel(nn.Module, SupportsVision):
466 inputs_embeds=inputs_embeds)
467 return hidden_states
468
469 - def compute_logits(self, hidden_states: torch.Tensor,
470 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
469 + def compute_logits(
470 + self,
471 + hidden_states: torch.Tensor,
472 + sampling_metadata: SamplingMetadata,
473 + ) -> Optional[torch.Tensor]:
474 return self.language_model.compute_logits(hidden_states,
475 sampling_metadata)
476
vllm/model_executor/models/jais.py
+5 -2
@@ -295,8 +295,11 @@ class JAISLMHeadModel(nn.Module):
295 attn_metadata)
296 return hidden_states
297
298 - def compute_logits(self, hidden_states: torch.Tensor,
299 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
298 + def compute_logits(
299 + self,
300 + hidden_states: torch.Tensor,
301 + sampling_metadata: SamplingMetadata,
302 + ) -> Optional[torch.Tensor]:
303 logits = self.logits_processor(self.lm_head, hidden_states,
304 sampling_metadata)
305 return logits
vllm/model_executor/models/jamba.py
+5 -2
@@ -861,8 +861,11 @@ class JambaForCausalLM(nn.Module, HasInnerState):
861 dtype=dtype,
862 device="cuda"))
863
864 - def compute_logits(self, hidden_states: torch.Tensor,
865 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
864 + def compute_logits(
865 + self,
866 + hidden_states: torch.Tensor,
867 + sampling_metadata: SamplingMetadata,
868 + ) -> Optional[torch.Tensor]:
869 logits = self.logits_processor(self.lm_head, hidden_states,
870 sampling_metadata)
871 return logits
vllm/model_executor/models/llama.py
+5 -2
@@ -430,8 +430,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
430 attn_metadata, intermediate_tensors)
431 return model_output
432
433 - def compute_logits(self, hidden_states: torch.Tensor,
434 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
433 + def compute_logits(
434 + self,
435 + hidden_states: torch.Tensor,
436 + sampling_metadata: SamplingMetadata,
437 + ) -> Optional[torch.Tensor]:
438 logits = self.logits_processor(self.lm_head, hidden_states,
439 sampling_metadata)
440 return logits
vllm/model_executor/models/llava.py
+5 -2
@@ -355,8 +355,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
355
356 return hidden_states
357
358 - def compute_logits(self, hidden_states: torch.Tensor,
359 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
358 + def compute_logits(
359 + self,
360 + hidden_states: torch.Tensor,
361 + sampling_metadata: SamplingMetadata,
362 + ) -> Optional[torch.Tensor]:
363 return self.language_model.compute_logits(hidden_states,
364 sampling_metadata)
365
vllm/model_executor/models/llava_next.py
+5 -2
@@ -588,8 +588,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
588
589 return hidden_states
590
591 - def compute_logits(self, hidden_states: torch.Tensor,
592 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
591 + def compute_logits(
592 + self,
593 + hidden_states: torch.Tensor,
594 + sampling_metadata: SamplingMetadata,
595 + ) -> Optional[torch.Tensor]:
596 return self.language_model.compute_logits(hidden_states,
597 sampling_metadata)
598
vllm/model_executor/models/medusa.py
+11 -5
@@ -65,22 +65,28 @@ class Medusa(nn.Module):
65 def compute_logits(
66 self, hidden_states: List[torch.Tensor],
67 sampling_metadata: SamplingMetadata) -> List[torch.Tensor]:
68 - logits = []
68 + logits_lst: List[torch.Tensor] = []
69
70 for hs, lm_head in zip(hidden_states, self.lm_heads):
71 _logits = self.logits_processor(lm_head, hs, sampling_metadata)
72
73 + if _logits is None:
74 + # _logits should only be None on rank > 0, in which case
75 + # it should remain true for every lm_head
76 + assert len(logits_lst) == 0
77 + continue
78 +
79 if self.token_map is None:
74 - logits.append(_logits)
80 + logits_lst.append(_logits)
81 else:
76 - logits.append(-torch.inf * torch.ones(
82 + logits_lst.append(-torch.inf * torch.ones(
83 size=(*_logits.shape[:-1], self.orig_vocab_size),
84 device=_logits.device,
85 dtype=_logits.dtype))
86
81 - logits[-1][..., self.token_map] = _logits
87 + logits_lst[-1][..., self.token_map] = _logits
88
83 - return logits
89 + return logits_lst
90
91 def sample(
92 self,
vllm/model_executor/models/minicpm.py
+5 -2
@@ -470,8 +470,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
470 attn_metadata, intermediate_tensors)
471 return hidden_states
472
473 - def compute_logits(self, hidden_states: torch.Tensor,
474 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
473 + def compute_logits(
474 + self,
475 + hidden_states: torch.Tensor,
476 + sampling_metadata: SamplingMetadata,
477 + ) -> Optional[torch.Tensor]:
478 hidden_states = hidden_states / self.scale_width
479 if self.config.tie_word_embeddings:
480 lm_head = self.model.embed_tokens
vllm/model_executor/models/minicpmv.py
+5 -2
@@ -630,8 +630,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsVision):
630 )
631 return output
632
633 - def compute_logits(self, hidden_states: torch.Tensor,
634 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
633 + def compute_logits(
634 + self,
635 + hidden_states: torch.Tensor,
636 + sampling_metadata: SamplingMetadata,
637 + ) -> Optional[torch.Tensor]:
638 logits = self.logits_processor(self.lm_head, hidden_states,
639 sampling_metadata)
640 return logits
vllm/model_executor/models/mixtral.py
+5 -2
@@ -375,8 +375,11 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
375 attn_metadata, intermediate_tensors)
376 return hidden_states
377
378 - def compute_logits(self, hidden_states: torch.Tensor,
379 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
378 + def compute_logits(
379 + self,
380 + hidden_states: torch.Tensor,
381 + sampling_metadata: SamplingMetadata,
382 + ) -> Optional[torch.Tensor]:
383 logits = self.logits_processor(self.lm_head, hidden_states,
384 sampling_metadata)
385 return logits
vllm/model_executor/models/mixtral_quant.py
+5 -2
@@ -362,8 +362,11 @@ class MixtralForCausalLM(nn.Module):
362 attn_metadata)
363 return hidden_states
364
365 - def compute_logits(self, hidden_states: torch.Tensor,
366 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
365 + def compute_logits(
366 + self,
367 + hidden_states: torch.Tensor,
368 + sampling_metadata: SamplingMetadata,
369 + ) -> Optional[torch.Tensor]:
370 logits = self.logits_processor(self.lm_head, hidden_states,
371 sampling_metadata)
372 return logits
vllm/model_executor/models/mpt.py
+5 -2
@@ -279,8 +279,11 @@ class MPTForCausalLM(nn.Module):
279 attn_metadata)
280 return hidden_states
281
282 - def compute_logits(self, hidden_states: torch.Tensor,
283 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
282 + def compute_logits(
283 + self,
284 + hidden_states: torch.Tensor,
285 + sampling_metadata: SamplingMetadata,
286 + ) -> Optional[torch.Tensor]:
287 logits = self.logits_processor(self.lm_head, hidden_states,
288 sampling_metadata)
289 return logits
vllm/model_executor/models/nemotron.py
+5 -2
@@ -453,8 +453,11 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA):
453 attn_metadata, intermediate_tensors)
454 return model_output
455
456 - def compute_logits(self, hidden_states: torch.Tensor,
457 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
456 + def compute_logits(
457 + self,
458 + hidden_states: torch.Tensor,
459 + sampling_metadata: SamplingMetadata,
460 + ) -> Optional[torch.Tensor]:
461 logits = self.logits_processor(self.lm_head, hidden_states,
462 sampling_metadata)
463 return logits
vllm/model_executor/models/olmo.py
+5 -2
@@ -311,8 +311,11 @@ class OlmoForCausalLM(nn.Module):
311 )
312 return hidden_states
313
314 - def compute_logits(self, hidden_states: torch.Tensor,
315 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
314 + def compute_logits(
315 + self,
316 + hidden_states: torch.Tensor,
317 + sampling_metadata: SamplingMetadata,
318 + ) -> Optional[torch.Tensor]:
319 logits = self.logits_processor(self.lm_head, hidden_states,
320 sampling_metadata)
321 return logits
vllm/model_executor/models/opt.py
+5 -2
@@ -323,8 +323,11 @@ class OPTForCausalLM(nn.Module):
323 attn_metadata)
324 return hidden_states
325
326 - def compute_logits(self, hidden_states: torch.Tensor,
327 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
326 + def compute_logits(
327 + self,
328 + hidden_states: torch.Tensor,
329 + sampling_metadata: SamplingMetadata,
330 + ) -> Optional[torch.Tensor]:
331 logits = self.logits_processor(self.lm_head, hidden_states,
332 sampling_metadata)
333 return logits
vllm/model_executor/models/orion.py
+5 -2
@@ -277,8 +277,11 @@ class OrionForCausalLM(nn.Module):
277 attn_metadata)
278 return hidden_states
279
280 - def compute_logits(self, hidden_states: torch.Tensor,
281 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
280 + def compute_logits(
281 + self,
282 + hidden_states: torch.Tensor,
283 + sampling_metadata: SamplingMetadata,
284 + ) -> Optional[torch.Tensor]:
285 logits = self.logits_processor(self.lm_head, hidden_states,
286 sampling_metadata)
287 return logits
vllm/model_executor/models/paligemma.py
+5 -2
@@ -262,8 +262,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
262 return hidden_states
263
264 # Copied from vllm/model_executor/models/gemma.py
265 - def compute_logits(self, hidden_states: torch.Tensor,
266 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
265 + def compute_logits(
266 + self,
267 + hidden_states: torch.Tensor,
268 + sampling_metadata: SamplingMetadata,
269 + ) -> Optional[torch.Tensor]:
270 logits = self.logits_processor(self.language_model.embed_tokens,
271 hidden_states, sampling_metadata)
272 return logits
vllm/model_executor/models/persimmon.py
+5 -2
@@ -285,8 +285,11 @@ class PersimmonForCausalLM(nn.Module):
285 )
286 return hidden_states
287
288 - def compute_logits(self, hidden_states: torch.Tensor,
289 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
288 + def compute_logits(
289 + self,
290 + hidden_states: torch.Tensor,
291 + sampling_metadata: SamplingMetadata,
292 + ) -> Optional[torch.Tensor]:
293 logits = self.logits_processor(self.lm_head, hidden_states,
294 sampling_metadata)
295 return logits
vllm/model_executor/models/phi.py
+5 -2
@@ -286,8 +286,11 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
286
287 return hidden_states
288
289 - def compute_logits(self, hidden_states: torch.Tensor,
290 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
289 + def compute_logits(
290 + self,
291 + hidden_states: torch.Tensor,
292 + sampling_metadata: SamplingMetadata,
293 + ) -> Optional[torch.Tensor]:
294 logits = self.logits_processor(self.lm_head, hidden_states,
295 sampling_metadata, self.lm_head.bias)
296 return logits
vllm/model_executor/models/phi3_small.py
+5 -2
@@ -399,8 +399,11 @@ class Phi3SmallForCausalLM(nn.Module):
399 def get_decoder(self):
400 return self.model
401
402 - def compute_logits(self, hidden_states: torch.Tensor,
403 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
402 + def compute_logits(
403 + self,
404 + hidden_states: torch.Tensor,
405 + sampling_metadata: SamplingMetadata,
406 + ) -> Optional[torch.Tensor]:
407 logits = self.logits_processor(self.lm_head, hidden_states,
408 sampling_metadata)
409 if self.dummy_token_indices is not None and logits is not None:
vllm/model_executor/models/phi3v.py
+5 -2
@@ -584,8 +584,11 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
584
585 return hidden_states
586
587 - def compute_logits(self, hidden_states: torch.Tensor,
588 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
587 + def compute_logits(
588 + self,
589 + hidden_states: torch.Tensor,
590 + sampling_metadata: SamplingMetadata,
591 + ) -> Optional[torch.Tensor]:
592 logits = self.logits_processor(self.lm_head, hidden_states,
593 sampling_metadata)
594 return logits
vllm/model_executor/models/qwen.py
+5 -2
@@ -281,8 +281,11 @@ class QWenLMHeadModel(nn.Module):
281 device=device),
282 })
283
284 - def compute_logits(self, hidden_states: torch.Tensor,
285 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
284 + def compute_logits(
285 + self,
286 + hidden_states: torch.Tensor,
287 + sampling_metadata: SamplingMetadata,
288 + ) -> Optional[torch.Tensor]:
289 logits = self.logits_processor(self.lm_head, hidden_states,
290 sampling_metadata)
291 return logits
vllm/model_executor/models/qwen2.py
+5 -2
@@ -362,8 +362,11 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
362 attn_metadata, intermediate_tensors)
363 return hidden_states
364
365 - def compute_logits(self, hidden_states: torch.Tensor,
366 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
365 + def compute_logits(
366 + self,
367 + hidden_states: torch.Tensor,
368 + sampling_metadata: SamplingMetadata,
369 + ) -> Optional[torch.Tensor]:
370 logits = self.logits_processor(self.lm_head, hidden_states,
371 sampling_metadata)
372 return logits
vllm/model_executor/models/qwen2_moe.py
+5 -2
@@ -400,8 +400,11 @@ class Qwen2MoeForCausalLM(nn.Module):
400 attn_metadata, intermediate_tensors)
401 return hidden_states
402
403 - def compute_logits(self, hidden_states: torch.Tensor,
404 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
403 + def compute_logits(
404 + self,
405 + hidden_states: torch.Tensor,
406 + sampling_metadata: SamplingMetadata,
407 + ) -> Optional[torch.Tensor]:
408 logits = self.logits_processor(self.lm_head, hidden_states,
409 sampling_metadata)
410 return logits
vllm/model_executor/models/stablelm.py
+5 -2
@@ -258,8 +258,11 @@ class StablelmForCausalLM(nn.Module):
258 attn_metadata)
259 return hidden_states
260
261 - def compute_logits(self, hidden_states: torch.Tensor,
262 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
261 + def compute_logits(
262 + self,
263 + hidden_states: torch.Tensor,
264 + sampling_metadata: SamplingMetadata,
265 + ) -> Optional[torch.Tensor]:
266 logits = self.logits_processor(self.lm_head, hidden_states,
267 sampling_metadata)
268 return logits
vllm/model_executor/models/starcoder2.py
+5 -2
@@ -268,8 +268,11 @@ class Starcoder2ForCausalLM(nn.Module):
268 attn_metadata)
269 return hidden_states
270
271 - def compute_logits(self, hidden_states: torch.Tensor,
272 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
271 + def compute_logits(
272 + self,
273 + hidden_states: torch.Tensor,
274 + sampling_metadata: SamplingMetadata,
275 + ) -> Optional[torch.Tensor]:
276 logits = self.logits_processor(self.lm_head, hidden_states,
277 sampling_metadata)
278 return logits
vllm/model_executor/models/xverse.py
+5 -2
@@ -328,8 +328,11 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
328 attn_metadata)
329 return hidden_states
330
331 - def compute_logits(self, hidden_states: torch.Tensor,
332 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
331 + def compute_logits(
332 + self,
333 + hidden_states: torch.Tensor,
334 + sampling_metadata: SamplingMetadata,
335 + ) -> Optional[torch.Tensor]:
336 logits = self.logits_processor(self.lm_head, hidden_states,
337 sampling_metadata)
338 return logits
vllm/outputs.py
+5 -3
@@ -1,6 +1,8 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import List, Optional, Tuple, Union
3 +from typing import List, Optional
4 +from typing import Sequence as GenericSequence
5 +from typing import Union
6
7 from vllm.lora.request import LoRARequest
8 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -28,7 +30,7 @@ class CompletionOutput:
30
31 index: int
32 text: str
31 - token_ids: Tuple[int, ...]
33 + token_ids: GenericSequence[int]
34 cumulative_logprob: Optional[float]
35 logprobs: Optional[SampleLogprobs]
36 finish_reason: Optional[str] = None
@@ -139,7 +141,7 @@ class RequestOutput:
141 CompletionOutput(
142 seqs.index(seq),
143 seq.get_output_text_to_return(text_buffer_length),
142 - seq.data._output_token_ids, # type: ignore
144 + seq.data._output_token_ids,
145 seq.get_cumulative_logprob() if include_logprobs else None,
146 seq.output_logprobs if include_logprobs else None,
147 SequenceStatus.get_finished_reason(seq.status),