b411418ff0 · tduncan/Nvidia-Optimization-Stack-VLLM

tests/spec_decode/test_scorer.py

+2 -3

@@ -62,9 +62,8 @@ def test_scorer(model_name: str, batch_size: int, max_propose_len: int,

62 scorer_worker = create_worker(Worker, model_name, block_size,

63 num_gpu_blocks, seed)

64 scorer_worker.model_runner.disable_logprobs = True # accessed by mqa_scorer

65 - scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor = True

66 - scorer_worker.model_runner.model.sampler.\

67 - should_modify_greedy_probs_inplace = True

65 + scorer_worker.model_runner.sampler.include_gpu_probs_tensor = True

66 + scorer_worker.model_runner.sampler.should_modify_greedy_probs_inplace = True

67

68 vocab_size = scorer_worker.vocab_size

69

vllm/model_executor/models/arctic.py

-10

@@ -24,7 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

24 from vllm.model_executor.layers.quantization.deepspeedfp import (

25 DeepSpeedFPConfig, DeepSpeedFPParameter)

26 from vllm.model_executor.layers.rotary_embedding import get_rope

27 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

27 from vllm.model_executor.layers.vocab_parallel_embedding import (

28 ParallelLMHead, VocabParallelEmbedding)

29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -435,7 +434,6 @@ class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):

434 self.unpadded_vocab_size = config.vocab_size

435 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

436 config.vocab_size)

438 - self.sampler = get_sampler()

437 self.make_empty_intermediate_tensors = (

438 self.model.make_empty_intermediate_tensors)

439

@@ -462,14 +460,6 @@ class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):

460 sampling_metadata)

461 return logits

462

465 - def sample(

466 - self,

467 - logits: Optional[torch.Tensor],

468 - sampling_metadata: SamplingMetadata,

469 - ) -> Optional[SamplerOutput]:

470 - next_tokens = self.sampler(logits, sampling_metadata)

471 - return next_tokens

472 -

463 def load_weights(self, weights: Iterable[Tuple[str,

464 torch.Tensor]]) -> Set[str]:

465 stacked_params_mapping = [

vllm/model_executor/models/aria.py

+1 -11

@@ -15,11 +15,10 @@ from vllm.model_executor.layers.fused_moe import FusedMoE

15 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

16 RowParallelLinear)

17 from vllm.model_executor.layers.logits_processor import LogitsProcessor

18 -from vllm.model_executor.layers.sampler import (SamplerOutput,

19 - SamplingMetadata, get_sampler)

18 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

19 from vllm.model_executor.model_loader.weight_utils import (

20 default_weight_loader, maybe_remap_kv_scale_name)

21 +from vllm.model_executor.sampling_metadata import SamplingMetadata

22 from vllm.multimodal import MULTIMODAL_REGISTRY

23 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

24 MultiModalKwargs)

@@ -527,7 +526,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

526 logit_scale = getattr(config, "logit_scale", 1.0)

527 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

528 self.vocab_size, logit_scale)

530 - self.sampler = get_sampler()

529

530 def _validate_image_sizes(

531 self, images: List[torch.Tensor]) -> List[torch.Tensor]:

@@ -653,14 +651,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

651 sampling_metadata)

652 return logits

653

656 - def sample(

657 - self,

658 - logits: torch.Tensor,

659 - sampling_metadata: SamplingMetadata,

660 - ) -> Optional[SamplerOutput]:

661 - next_tokens = self.sampler(logits, sampling_metadata)

662 - return next_tokens

663 -

654 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

655 loader = AutoWeightsLoader(self)

656 loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

vllm/model_executor/models/aya_vision.py

-16

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0 Adapted from

2 # https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision

3 -from functools import cached_property

3 from typing import (Iterable, Literal, Mapping, Optional, Sequence, Set, Tuple,

4 TypedDict, Union, cast)

5

@@ -17,7 +16,6 @@ from transformers.models.got_ocr2.image_processing_got_ocr2 import (

16

17 from vllm.config import VllmConfig

18 from vllm.jsontree import json_map_leaves

20 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

19 from vllm.model_executor.sampling_metadata import SamplingMetadata

20 from vllm.multimodal import MULTIMODAL_REGISTRY

21 from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargs

@@ -461,17 +459,3 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

459 ) -> Optional[torch.Tensor]:

460 return self.language_model.compute_logits(hidden_states,

461 sampling_metadata)

464 -

465 - @cached_property

466 - def sampler(self):

467 - if hasattr(self.language_model, "sampler"):

468 - return self.language_model.sampler

469 -

470 - return get_sampler()

471 -

472 - def sample(

473 - self,

474 - logits: torch.Tensor,

475 - sampling_metadata: SamplingMetadata,

476 - ) -> Optional[SamplerOutput]:

477 - return self.language_model.sample(logits, sampling_metadata)

vllm/model_executor/models/baichuan.py

-10

@@ -39,7 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

39 from vllm.model_executor.layers.logits_processor import LogitsProcessor

40 from vllm.model_executor.layers.quantization import QuantizationConfig

41 from vllm.model_executor.layers.rotary_embedding import get_rope

42 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

42 from vllm.model_executor.layers.vocab_parallel_embedding import (

43 ParallelLMHead, VocabParallelEmbedding)

44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -396,7 +395,6 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP,

395 if self.config.tie_word_embeddings:

396 self.lm_head.weight = self.model.embed_tokens.weight

397 self.logits_processor = LogitsProcessor(config.vocab_size)

399 - self.sampler = get_sampler()

398 self.make_empty_intermediate_tensors = (

399 self.model.make_empty_intermediate_tensors)

400

@@ -423,14 +421,6 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP,

421 sampling_metadata)

422 return logits

423

426 - def sample(

427 - self,

428 - logits: torch.Tensor,

429 - sampling_metadata: SamplingMetadata,

430 - ) -> Optional[SamplerOutput]:

431 - next_tokens = self.sampler(logits, sampling_metadata)

432 - return next_tokens

433 -

424 def load_weights(self, weights: Iterable[Tuple[str,

425 torch.Tensor]]) -> Set[str]:

426 loader = AutoWeightsLoader(self)

vllm/model_executor/models/bamba.py

-10

@@ -24,7 +24,6 @@ from vllm.model_executor.layers.mamba.mamba_mixer2 import (

24 MambaMixer2, extra_groups_for_head_shards)

25 from vllm.model_executor.layers.quantization import QuantizationConfig

26 from vllm.model_executor.layers.rotary_embedding import get_rope

27 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

27 from vllm.model_executor.layers.vocab_parallel_embedding import (

28 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -462,7 +461,6 @@ class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,

461

462 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

463 config.vocab_size)

465 - self.sampler = get_sampler()

464

465 self.make_empty_intermediate_tensors = (

466 self.model.make_empty_intermediate_tensors)

@@ -538,14 +536,6 @@ class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,

536 sampling_metadata)

537 return logits

538

541 - def sample(

542 - self,

543 - logits: Optional[torch.Tensor],

544 - sampling_metadata: SamplingMetadata,

545 - ) -> Optional[SamplerOutput]:

546 - next_tokens = self.sampler(logits, sampling_metadata)

547 - return next_tokens

548 -

539 def load_weights(self, weights: Iterable[Tuple[str,

540 torch.Tensor]]) -> Set[str]:

541 loader = AutoWeightsLoader(self)

vllm/model_executor/models/bart.py

-10

@@ -37,7 +37,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

37 from vllm.model_executor.layers.logits_processor import LogitsProcessor

38 from vllm.model_executor.layers.quantization.base_config import (

39 QuantizationConfig)

40 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

40 from vllm.model_executor.layers.vocab_parallel_embedding import (

41 ParallelLMHead, VocabParallelEmbedding)

42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -791,7 +790,6 @@ class BartForConditionalGeneration(nn.Module, SupportsV0Only, SupportsQuant):

790

791 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

792 config.vocab_size)

794 - self.sampler = get_sampler()

793

794 def forward(

795 self,

@@ -828,14 +826,6 @@ class BartForConditionalGeneration(nn.Module, SupportsV0Only, SupportsQuant):

826 sampling_metadata)

827 return logits

828

831 - def sample(

832 - self,

833 - logits: Optional[torch.Tensor],

834 - sampling_metadata: SamplingMetadata,

835 - ) -> Optional[SamplerOutput]:

836 - next_tokens = self.sampler(logits, sampling_metadata)

837 - return next_tokens

838 -

829 stacked_params_mapping = {

830 "q_proj": {

831 "param_name": "qkv_proj",

vllm/model_executor/models/blip2.py

+1 -17

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from collections.abc import Iterable, Mapping, Sequence

4 -from functools import cached_property

4 from typing import Literal, Optional, Set, Tuple, TypedDict, Union

5

6 import torch

@@ -12,7 +11,6 @@ from transformers import (BatchFeature, Blip2Config, Blip2QFormerConfig,

11 from vllm.config import CacheConfig, VllmConfig

12 from vllm.model_executor.layers.activation import get_act_fn

13 from vllm.model_executor.layers.quantization import QuantizationConfig

15 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

14 from vllm.model_executor.sampling_metadata import SamplingMetadata

15 from vllm.multimodal import MULTIMODAL_REGISTRY

16 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -530,13 +528,6 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

528 self.make_empty_intermediate_tensors = (

529 self.language_model.make_empty_intermediate_tensors)

530

533 - @cached_property

534 - def sampler(self):

535 - if hasattr(self.language_model, "sampler"):

536 - return self.language_model.sampler

537 -

538 - return get_sampler()

539 -

531 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:

532 h = w = self.config.vision_config.image_size

533 expected_dims = (3, h, w)

@@ -649,7 +640,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

640 intermediate_tensors: Optional[IntermediateTensors] = None,

641 inputs_embeds: Optional[torch.Tensor] = None,

642 **kwargs: object,

652 - ) -> Union[SamplerOutput, IntermediateTensors]:

643 + ) -> IntermediateTensors:

644 """Run forward pass for BLIP-2.

645

646 One key thing to understand is the `input_ids` already accounts for the

@@ -707,13 +698,6 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

698 return self.language_model.compute_logits(hidden_states,

699 sampling_metadata)

700

710 - def sample(

711 - self,

712 - logits: torch.Tensor,

713 - sampling_metadata: SamplingMetadata,

714 - ) -> Optional[SamplerOutput]:

715 - return self.language_model.sample(logits, sampling_metadata)

716 -

701 def load_weights(self, weights: Iterable[Tuple[str,

702 torch.Tensor]]) -> Set[str]:

703 loader = AutoWeightsLoader(self)

vllm/model_executor/models/bloom.py

-10

@@ -35,7 +35,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

35 RowParallelLinear)

36 from vllm.model_executor.layers.logits_processor import LogitsProcessor

37 from vllm.model_executor.layers.quantization import QuantizationConfig

38 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

38 from vllm.model_executor.layers.vocab_parallel_embedding import (

39 ParallelLMHead, VocabParallelEmbedding)

40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -297,7 +296,6 @@ class BloomForCausalLM(nn.Module, SupportsPP, SupportsV0Only, SupportsQuant):

296 self.config.hidden_size)

297

298 self.logits_processor = LogitsProcessor(config.vocab_size)

300 - self.sampler = get_sampler()

299 self.make_empty_intermediate_tensors = (

300 self.transformer.make_empty_intermediate_tensors)

301

@@ -324,14 +322,6 @@ class BloomForCausalLM(nn.Module, SupportsPP, SupportsV0Only, SupportsQuant):

322 sampling_metadata)

323 return logits

324

327 - def sample(

328 - self,

329 - logits: torch.Tensor,

330 - sampling_metadata: SamplingMetadata,

331 - ) -> Optional[SamplerOutput]:

332 - next_tokens = self.sampler(logits, sampling_metadata)

333 - return next_tokens

334 -

325 def load_weights(self, weights: Iterable[Tuple[str,

326 torch.Tensor]]) -> Set[str]:

327 params_dict = dict(self.named_parameters(remove_duplicate=False))

vllm/model_executor/models/chameleon.py

-10

@@ -22,7 +22,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

22 from vllm.model_executor.layers.logits_processor import LogitsProcessor

23 from vllm.model_executor.layers.quantization import QuantizationConfig

24 from vllm.model_executor.layers.rotary_embedding import get_rope

25 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

25 from vllm.model_executor.layers.vocab_parallel_embedding import (

26 ParallelLMHead, VocabParallelEmbedding)

27 from vllm.model_executor.model_loader.weight_utils import (

@@ -950,7 +949,6 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

949 logit_scale = getattr(config, "logit_scale", 1.0)

950 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

951 config.vocab_size, logit_scale)

953 - self.sampler = get_sampler()

952 self.make_empty_intermediate_tensors = (

953 self.model.make_empty_intermediate_tensors)

954

@@ -1054,14 +1052,6 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

1052

1053 return logits

1054

1057 - def sample(

1058 - self,

1059 - logits: torch.Tensor,

1060 - sampling_metadata: SamplingMetadata,

1061 - ) -> Optional[SamplerOutput]:

1062 - next_tokens = self.sampler(logits, sampling_metadata)

1063 - return next_tokens

1064 -

1055 def load_weights(self, weights: Iterable[Tuple[str,

1056 torch.Tensor]]) -> Set[str]:

1057 stacked_params_mapping = [

vllm/model_executor/models/chatglm.py

-10

@@ -21,7 +21,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

21 from vllm.model_executor.layers.logits_processor import LogitsProcessor

22 from vllm.model_executor.layers.quantization import QuantizationConfig

23 from vllm.model_executor.layers.rotary_embedding import get_rope

24 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

24 from vllm.model_executor.layers.vocab_parallel_embedding import (

25 ParallelLMHead, VocabParallelEmbedding)

26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -429,7 +428,6 @@ class ChatGLMBaseModel(nn.Module):

428 self.transformer.embedding.weight)

429 self.lm_head = self.transformer.output_layer

430 self.logits_processor = LogitsProcessor(config.padded_vocab_size)

432 - self.sampler = get_sampler()

431 self.make_empty_intermediate_tensors = (

432 self.transformer.make_empty_intermediate_tensors)

433

@@ -442,14 +440,6 @@ class ChatGLMBaseModel(nn.Module):

440 sampling_metadata)

441 return logits

442

445 - def sample(

446 - self,

447 - logits: torch.Tensor,

448 - sampling_metadata: SamplingMetadata,

449 - ) -> Optional[SamplerOutput]:

450 - next_tokens = self.sampler(logits, sampling_metadata)

451 - return next_tokens

452 -

443 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

444 loader = AutoWeightsLoader(self)

445 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

vllm/model_executor/models/commandr.py

-10

@@ -38,7 +38,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

38 from vllm.model_executor.layers.logits_processor import LogitsProcessor

39 from vllm.model_executor.layers.quantization import QuantizationConfig

40 from vllm.model_executor.layers.rotary_embedding import get_rope

41 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

41 from vllm.model_executor.layers.vocab_parallel_embedding import (

42 VocabParallelEmbedding)

43 from vllm.model_executor.model_loader.weight_utils import (

@@ -372,7 +371,6 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):

371 scale=config.logit_scale)

372 self.model = CohereModel(vllm_config=vllm_config,

373 prefix=maybe_prefix(prefix, "model"))

375 - self.sampler = get_sampler()

374 self.make_empty_intermediate_tensors = (

375 self.model.make_empty_intermediate_tensors)

376

@@ -406,14 +404,6 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):

404

405 return logits

406

409 - def sample(

410 - self,

411 - logits: torch.Tensor,

412 - sampling_metadata: SamplingMetadata,

413 - ) -> Optional[SamplerOutput]:

414 - next_tokens = self.sampler(logits, sampling_metadata)

415 - return next_tokens

416 -

407 def load_weights(self, weights: Iterable[Tuple[str,

408 torch.Tensor]]) -> Set[str]:

409 stacked_params_mapping = [

vllm/model_executor/models/dbrx.py

-10

@@ -16,7 +16,6 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

16 from vllm.model_executor.layers.logits_processor import LogitsProcessor

17 from vllm.model_executor.layers.quantization import QuantizationConfig

18 from vllm.model_executor.layers.rotary_embedding import get_rope

19 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

19 from vllm.model_executor.layers.vocab_parallel_embedding import (

20 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

21 from vllm.model_executor.model_loader.weight_utils import (

@@ -390,7 +389,6 @@ class DbrxForCausalLM(nn.Module, SupportsPP):

389 )

390 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

391 config.vocab_size)

393 - self.sampler = get_sampler()

392 self.make_empty_intermediate_tensors = (

393 self.transformer.make_empty_intermediate_tensors)

394

@@ -417,14 +415,6 @@ class DbrxForCausalLM(nn.Module, SupportsPP):

415 sampling_metadata)

416 return logits

417

420 - def sample(

421 - self,

422 - logits: Optional[torch.Tensor],

423 - sampling_metadata: SamplingMetadata,

424 - ) -> Optional[SamplerOutput]:

425 - next_tokens = self.sampler(logits, sampling_metadata)

426 - return next_tokens

427 -

418 def load_weights(self, weights: Iterable[Tuple[str,

419 torch.Tensor]]) -> Set[str]:

420 expert_params_mapping = [(

vllm/model_executor/models/deepseek.py

-10

@@ -43,7 +43,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

43 from vllm.model_executor.layers.logits_processor import LogitsProcessor

44 from vllm.model_executor.layers.quantization import QuantizationConfig

45 from vllm.model_executor.layers.rotary_embedding import get_rope

46 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

46 from vllm.model_executor.layers.vocab_parallel_embedding import (

47 ParallelLMHead, VocabParallelEmbedding)

48 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -453,7 +452,6 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):

452 if self.config.tie_word_embeddings:

453 self.lm_head.weight = self.model.embed_tokens.weight

454 self.logits_processor = LogitsProcessor(config.vocab_size)

456 - self.sampler = get_sampler()

455 self.make_empty_intermediate_tensors = (

456 self.model.make_empty_intermediate_tensors)

457

@@ -480,14 +478,6 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):

478 sampling_metadata)

479 return logits

480

483 - def sample(

484 - self,

485 - logits: Optional[torch.Tensor],

486 - sampling_metadata: SamplingMetadata,

487 - ) -> Optional[SamplerOutput]:

488 - next_tokens = self.sampler(logits, sampling_metadata)

489 - return next_tokens

490 -

481 def load_weights(self, weights: Iterable[Tuple[str,

482 torch.Tensor]]) -> Set[str]:

483 loader = AutoWeightsLoader(self)

vllm/model_executor/models/deepseek_mtp.py

-11

@@ -10,7 +10,6 @@ from vllm.model_executor.layers.fused_moe import FusedMoE

10 from vllm.model_executor.layers.layernorm import RMSNorm

11 from vllm.model_executor.layers.logits_processor import LogitsProcessor

12 from vllm.model_executor.layers.quantization import QuantizationConfig

13 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

13 from vllm.model_executor.layers.vocab_parallel_embedding import (

14 ParallelLMHead, VocabParallelEmbedding)

15 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -154,8 +153,6 @@ class DeepSeekMTP(nn.Module):

153 prefix=maybe_prefix(

154 prefix, "model"))

155

157 - self.sampler = get_sampler()

158 -

156 def forward(

157 self,

158 input_ids: torch.Tensor,

@@ -179,14 +176,6 @@ class DeepSeekMTP(nn.Module):

176 return self.model.compute_logits(hidden_states, sampling_metadata,

177 spec_step_idx)

178

182 - def sample(

183 - self,

184 - logits: torch.Tensor,

185 - sampling_metadata: SamplingMetadata,

186 - ) -> Optional[SamplerOutput]:

187 - next_tokens = self.sampler(logits, sampling_metadata)

188 - return next_tokens

189 -

179 def load_weights(self, weights: Iterable[Tuple[str,

180 torch.Tensor]]) -> Set[str]:

181 stacked_params_mapping = [

vllm/model_executor/models/deepseek_v2.py

-10

@@ -44,7 +44,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

44 from vllm.model_executor.layers.logits_processor import LogitsProcessor

45 from vllm.model_executor.layers.quantization import QuantizationConfig

46 from vllm.model_executor.layers.rotary_embedding import get_rope

47 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

47 from vllm.model_executor.layers.vocab_parallel_embedding import (

48 ParallelLMHead, VocabParallelEmbedding)

49 from vllm.model_executor.model_loader.weight_utils import (

@@ -686,7 +685,6 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):

685 else:

686 self.lm_head = PPMissingLayer()

687 self.logits_processor = LogitsProcessor(config.vocab_size)

689 - self.sampler = get_sampler()

688 self.make_empty_intermediate_tensors = (

689 self.model.make_empty_intermediate_tensors)

690

@@ -713,14 +711,6 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):

711 sampling_metadata)

712 return logits

713

716 - def sample(

717 - self,

718 - logits: Optional[torch.Tensor],

719 - sampling_metadata: SamplingMetadata,

720 - ) -> Optional[SamplerOutput]:

721 - next_tokens = self.sampler(logits, sampling_metadata)

722 - return next_tokens

723 -

714 def make_empty_intermediate_tensors(

715 self, batch_size: int, dtype: torch.dtype,

716 device: torch.device) -> IntermediateTensors:

vllm/model_executor/models/deepseek_vl2.py

-16

@@ -4,7 +4,6 @@

4 """Inference-only Deepseek-VL2 model compatible with HuggingFace weights."""

5 import math

6 from collections.abc import Iterable, Mapping, Sequence

7 -from functools import cached_property

7 from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union

8

9 import torch

@@ -16,7 +15,6 @@ from transformers import BatchFeature

15 from vllm.config import VllmConfig

16 from vllm.model_executor import SamplingMetadata

17 from vllm.model_executor.layers.quantization import QuantizationConfig

19 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

18 from vllm.model_executor.model_loader.utils import set_default_torch_dtype

19 from vllm.multimodal import MULTIMODAL_REGISTRY

20 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -393,13 +391,6 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

391 model = model.to(dtype=torch.get_default_dtype())

392 return model

393

396 - @cached_property

397 - def sampler(self):

398 - if hasattr(self.language_model, "sampler"):

399 - return self.language_model.sampler

400 -

401 - return get_sampler()

402 -

394 def _validate_pixel_values(

395 self, data: Union[torch.Tensor, List[torch.Tensor]]

396 ) -> Union[torch.Tensor, List[torch.Tensor]]:

@@ -647,13 +638,6 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

638 return self.language_model.compute_logits(hidden_states,

639 sampling_metadata)

640

650 - def sample(

651 - self,

652 - logits: torch.Tensor,

653 - sampling_metadata: SamplingMetadata,

654 - ) -> Optional[SamplerOutput]:

655 - return self.language_model.sample(logits, sampling_metadata)

656 -

641 def load_weights(self, weights: Iterable[Tuple[str,

642 torch.Tensor]]) -> Set[str]:

643

vllm/model_executor/models/eagle.py

-13

@@ -9,7 +9,6 @@ from vllm.config import VllmConfig

9 from vllm.logger import init_logger

10 from vllm.model_executor.layers.layernorm import RMSNorm

11 from vllm.model_executor.layers.logits_processor import LogitsProcessor

12 -from vllm.model_executor.layers.sampler import SamplerOutput

12 from vllm.model_executor.layers.vocab_parallel_embedding import (

13 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)

14 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -131,10 +130,6 @@ class EAGLE(nn.Module):

130 # checkpoint file has token_map tensor.

131 self.token_map = None

132

134 - @property

135 - def sampler(self):

136 - return self.model.sampler

137 -

133 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:

134 return self.model.model.get_input_embeddings(input_ids)

135

@@ -188,14 +183,6 @@ class EAGLE(nn.Module):

183

184 return logits

185

191 - def sample(

192 - self,

193 - logits: torch.Tensor,

194 - sampling_metadata: SamplingMetadata,

195 - ) -> Optional[SamplerOutput]:

196 - next_tokens = self.sampler(logits, sampling_metadata)

197 - return next_tokens

198 -

186 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

187 # This implementation is incompitable with https://huggingface.co/yuhuili/EAGLE-LLaMA3-Instruct-8B

188 # due to missing lm_head weights and its config being that of a

vllm/model_executor/models/exaone.py

-11

@@ -41,7 +41,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

41 from vllm.model_executor.layers.logits_processor import LogitsProcessor

42 from vllm.model_executor.layers.quantization import QuantizationConfig

43 from vllm.model_executor.layers.rotary_embedding import get_rope

44 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

44 from vllm.model_executor.layers.vocab_parallel_embedding import (

45 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

46 from vllm.model_executor.model_loader.weight_utils import (

@@ -510,8 +509,6 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

509 else:

510 self.lm_head = PPMissingLayer()

511

513 - self.sampler = get_sampler()

514 -

512 self.make_empty_intermediate_tensors = (

513 self.transformer.make_empty_intermediate_tensors)

514

@@ -538,14 +535,6 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

535 sampling_metadata)

536 return logits

537

541 - def sample(

542 - self,

543 - logits: torch.Tensor,

544 - sampling_metadata: SamplingMetadata,

545 - ) -> Optional[SamplerOutput]:

546 - next_tokens = self.sampler(logits, sampling_metadata)

547 - return next_tokens

548 -

538 def load_weights(self, weights: Iterable[Tuple[str,

539 torch.Tensor]]) -> Set[str]:

540 loader = AutoWeightsLoader(

vllm/model_executor/models/falcon.py

-10

@@ -40,7 +40,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

40 from vllm.model_executor.layers.logits_processor import LogitsProcessor

41 from vllm.model_executor.layers.quantization import QuantizationConfig

42 from vllm.model_executor.layers.rotary_embedding import get_rope

43 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

43 from vllm.model_executor.layers.vocab_parallel_embedding import (

44 ParallelLMHead, VocabParallelEmbedding)

45 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -473,7 +472,6 @@ class FalconForCausalLM(nn.Module, SupportsPP):

472 quant_config=quant_config,

473 )

474 self.logits_processor = LogitsProcessor(config.vocab_size)

476 - self.sampler = get_sampler()

475 self.make_empty_intermediate_tensors = (

476 self.transformer.make_empty_intermediate_tensors)

477

@@ -500,14 +498,6 @@ class FalconForCausalLM(nn.Module, SupportsPP):

498 sampling_metadata)

499 return logits

500

503 - def sample(

504 - self,

505 - logits: torch.Tensor,

506 - sampling_metadata: SamplingMetadata,

507 - ) -> Optional[SamplerOutput]:

508 - next_tokens = self.sampler(logits, sampling_metadata)

509 - return next_tokens

510 -

501 def load_weights(self, weights: Iterable[Tuple[str,

502 torch.Tensor]]) -> Set[str]:

503 loader = AutoWeightsLoader(

vllm/model_executor/models/florence2.py

-21

@@ -3,7 +3,6 @@

3 import math

4 from collections import OrderedDict

5 from collections.abc import Iterable, Mapping, Sequence

6 -from functools import cached_property

6 from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union

7

8 import torch

@@ -14,7 +13,6 @@ from transformers import BartTokenizer, BatchFeature, PretrainedConfig

13

14 from vllm.config import VllmConfig

15 from vllm.model_executor.layers.logits_processor import LogitsProcessor

17 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

16 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

17 from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,

18 BartParallelLMHead,

@@ -673,7 +671,6 @@ class Florence2LanguageForConditionalGeneration(nn.Module, SupportsV0Only):

671

672 self.logits_processor = LogitsProcessor(self.vocab_size,

673 config.vocab_size)

676 - self.sampler = get_sampler()

674

675 def forward(

676 self,

@@ -716,11 +713,6 @@ class Florence2LanguageForConditionalGeneration(nn.Module, SupportsV0Only):

713 sampling_metadata)

714 return logits

715

719 - def sample(self, logits: torch.Tensor,

720 - sampling_metadata: SamplingMetadata) -> SamplerOutput:

721 - next_tokens = self.sampler(logits, sampling_metadata)

722 - return next_tokens

723 -

716 def load_weights(self, weights: Iterable[Tuple[str,

717 torch.Tensor]]) -> Set[str]:

718 stacked_params_mapping = [

@@ -929,12 +921,6 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

921 raise NotImplementedError(

922 'Florence2 only supports COSINE as temporal embedding.')

923

932 - @cached_property

933 - def sampler(self):

934 - if hasattr(self.language_model, "sampler"):

935 - return self.language_model.sampler

936 - return get_sampler()

937 -

924 def _validate_pixel_values(

925 self, data: Union[torch.Tensor, List[torch.Tensor]]

926 ) -> Union[torch.Tensor, List[torch.Tensor]]:

@@ -1110,13 +1096,6 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

1096 return self.language_model.compute_logits(hidden_states,

1097 sampling_metadata)

1098

1113 - def sample(

1114 - self,

1115 - logits: torch.Tensor,

1116 - sampling_metadata: SamplingMetadata,

1117 - ) -> SamplerOutput:

1118 - return self.language_model.sample(logits, sampling_metadata)

1119 -

1099 def load_weights(self, weights: Iterable[Tuple[str,

1100 torch.Tensor]]) -> Set[str]:

1101 loader = AutoWeightsLoader(self)

vllm/model_executor/models/fuyu.py

-13

@@ -27,7 +27,6 @@ from transformers import (BatchFeature, FuyuConfig, FuyuImageProcessor,

27

28 from vllm.config import VllmConfig

29 from vllm.model_executor.layers.linear import ColumnParallelLinear

30 -from vllm.model_executor.layers.sampler import SamplerOutput

30 from vllm.model_executor.models.persimmon import PersimmonForCausalLM

31 from vllm.model_executor.sampling_metadata import SamplingMetadata

32 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -270,10 +269,6 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

269 self.make_empty_intermediate_tensors = (

270 self.language_model.make_empty_intermediate_tensors)

271

273 - @property

274 - def sampler(self):

275 - return self.language_model.sampler

276 -

272 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:

273

274 h = w = self.config.patch_size

@@ -387,14 +382,6 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

382 self.language_model.lm_head, hidden_states, sampling_metadata)

383 return logits

384

390 - def sample(

391 - self,

392 - logits: torch.Tensor,

393 - sampling_metadata: SamplingMetadata,

394 - ) -> Optional[SamplerOutput]:

395 - next_tokens = self.language_model.sampler(logits, sampling_metadata)

396 - return next_tokens

397 -

385 def load_weights(self, weights: Iterable[Tuple[str,

386 torch.Tensor]]) -> Set[str]:

387 loader = AutoWeightsLoader(self)

vllm/model_executor/models/gemma.py

-10

@@ -35,7 +35,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

35 from vllm.model_executor.layers.logits_processor import LogitsProcessor

36 from vllm.model_executor.layers.quantization import QuantizationConfig

37 from vllm.model_executor.layers.rotary_embedding import get_rope

38 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

38 from vllm.model_executor.layers.vocab_parallel_embedding import (

39 VocabParallelEmbedding)

40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -388,7 +387,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

387 self.model = GemmaModel(vllm_config=vllm_config,

388 prefix=maybe_prefix(prefix, "model"))

389 self.logits_processor = LogitsProcessor(config.vocab_size)

391 - self.sampler = get_sampler()

390 self.make_empty_intermediate_tensors = (

391 self.model.make_empty_intermediate_tensors)

392

@@ -415,14 +413,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

413 sampling_metadata)

414 return logits

415

418 - def sample(

419 - self,

420 - logits: torch.Tensor,

421 - sampling_metadata: SamplingMetadata,

422 - ) -> Optional[SamplerOutput]:

423 - next_tokens = self.sampler(logits, sampling_metadata)

424 - return next_tokens

425 -

416 def load_weights(self, weights: Iterable[Tuple[str,

417 torch.Tensor]]) -> Set[str]:

418 loader = AutoWeightsLoader(

vllm/model_executor/models/gemma2.py

-10

@@ -34,7 +34,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

34 from vllm.model_executor.layers.logits_processor import LogitsProcessor

35 from vllm.model_executor.layers.quantization import QuantizationConfig

36 from vllm.model_executor.layers.rotary_embedding import get_rope

37 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

37 from vllm.model_executor.layers.vocab_parallel_embedding import (

38 VocabParallelEmbedding)

39 from vllm.model_executor.model_loader.weight_utils import (

@@ -388,7 +387,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

387 prefix=maybe_prefix(prefix, "model"))

388 self.logits_processor = LogitsProcessor(

389 config.vocab_size, soft_cap=config.final_logit_softcapping)

391 - self.sampler = get_sampler()

390 self.make_empty_intermediate_tensors = (

391 self.model.make_empty_intermediate_tensors)

392

@@ -415,14 +413,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

413 sampling_metadata)

414 return logits

415

418 - def sample(

419 - self,

420 - logits: torch.Tensor,

421 - sampling_metadata: SamplingMetadata,

422 - ) -> Optional[SamplerOutput]:

423 - next_tokens = self.sampler(logits, sampling_metadata)

424 - return next_tokens

425 -

416 def load_weights(self, weights: Iterable[Tuple[str,

417 torch.Tensor]]) -> Set[str]:

418 loader = AutoWeightsLoader(

vllm/model_executor/models/gemma3.py

-10

@@ -34,7 +34,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

34 from vllm.model_executor.layers.logits_processor import LogitsProcessor

35 from vllm.model_executor.layers.quantization import QuantizationConfig

36 from vllm.model_executor.layers.rotary_embedding import get_rope

37 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

37 from vllm.model_executor.layers.vocab_parallel_embedding import (

38 VocabParallelEmbedding)

39 from vllm.model_executor.model_loader.weight_utils import (

@@ -493,7 +492,6 @@ class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

492 prefix=maybe_prefix(prefix, "model"))

493 self.logits_processor = LogitsProcessor(

494 config.vocab_size, soft_cap=config.final_logit_softcapping)

496 - self.sampler = get_sampler()

495 self.make_empty_intermediate_tensors = (

496 self.model.make_empty_intermediate_tensors)

497

@@ -521,14 +519,6 @@ class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

519 sampling_metadata)

520 return logits

521

524 - def sample(

525 - self,

526 - logits: torch.Tensor,

527 - sampling_metadata: SamplingMetadata,

528 - ) -> Optional[SamplerOutput]:

529 - next_tokens = self.sampler(logits, sampling_metadata)

530 - return next_tokens

531 -

522 def load_weights(self, weights: Iterable[Tuple[str,

523 torch.Tensor]]) -> Set[str]:

524 loader = AutoWeightsLoader(

vllm/model_executor/models/gemma3_mm.py

+2 -14

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 import math

3 from collections.abc import Iterable, Mapping, Sequence

4 -from typing import Any, Literal, Optional, Set, Tuple, TypedDict, Union

4 +from typing import Any, Literal, Optional, Set, Tuple, TypedDict

5

6 import torch

7 from torch import nn

@@ -12,7 +12,6 @@ import vllm.envs as envs

12 from vllm.config import VllmConfig

13 from vllm.logger import init_logger

14 from vllm.model_executor.layers.layernorm import GemmaRMSNorm

15 -from vllm.model_executor.layers.sampler import SamplerOutput

15 from vllm.model_executor.models.module_mapping import MultiModelKeys

16 from vllm.model_executor.sampling_metadata import SamplingMetadata

17 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -503,10 +502,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

502 def dtype(self):

503 return next(self.parameters()).dtype

504

506 - @property

507 - def sampler(self):

508 - return self.language_model.sampler

509 -

505 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:

506 h = w = self.config.vision_config.image_size

507 expected_dims = (3, h, w)

@@ -607,7 +602,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

602 positions: torch.Tensor,

603 intermediate_tensors: Optional[IntermediateTensors] = None,

604 inputs_embeds: Optional[torch.Tensor] = None,

610 - **kwargs: object) -> Union[SamplerOutput, IntermediateTensors]:

605 + **kwargs: object) -> IntermediateTensors:

606 if intermediate_tensors is not None:

607 inputs_embeds = None

608

@@ -704,13 +699,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

699 return self.language_model.compute_logits(hidden_states,

700 sampling_metadata)

701

707 - def sample(

708 - self,

709 - logits: torch.Tensor,

710 - sampling_metadata: SamplingMetadata,

711 - ) -> Optional[SamplerOutput]:

712 - return self.language_model.sample(logits, sampling_metadata)

713 -

702 def load_weights(self, weights: Iterable[Tuple[str,

703 torch.Tensor]]) -> Set[str]:

704 loader = AutoWeightsLoader(self)

vllm/model_executor/models/glm4.py

-10

@@ -37,7 +37,6 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

37 from vllm.model_executor.layers.logits_processor import LogitsProcessor

38 from vllm.model_executor.layers.quantization import QuantizationConfig

39 from vllm.model_executor.layers.rotary_embedding import get_rope

40 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

40 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

41 from vllm.model_executor.sampling_metadata import SamplingMetadata

42 from vllm.sequence import IntermediateTensors

@@ -267,7 +266,6 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

266 self.lm_head = PPMissingLayer()

267

268 self.logits_processor = LogitsProcessor(config.vocab_size)

270 - self.sampler = get_sampler()

269

270 self.make_empty_intermediate_tensors = (

271 self.model.make_empty_intermediate_tensors)

@@ -295,14 +293,6 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

293 sampling_metadata)

294 return logits

295

298 - def sample(

299 - self,

300 - logits: torch.Tensor,

301 - sampling_metadata: SamplingMetadata,

302 - ) -> Optional[SamplerOutput]:

303 - next_tokens = self.sampler(logits, sampling_metadata)

304 - return next_tokens

305 -

296 def load_weights(self, weights: Iterable[Tuple[str,

297 torch.Tensor]]) -> Set[str]:

298 loader = AutoWeightsLoader(

vllm/model_executor/models/gpt2.py

-10

@@ -35,7 +35,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

35 RowParallelLinear)

36 from vllm.model_executor.layers.logits_processor import LogitsProcessor

37 from vllm.model_executor.layers.quantization import QuantizationConfig

38 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

38 from vllm.model_executor.layers.vocab_parallel_embedding import (

39 ParallelLMHead, VocabParallelEmbedding)

40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -255,7 +254,6 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):

254 self.lm_head = self.lm_head.tie_weights(self.transformer.wte)

255

256 self.logits_processor = LogitsProcessor(config.vocab_size)

258 - self.sampler = get_sampler()

257 self.make_empty_intermediate_tensors = (

258 self.transformer.make_empty_intermediate_tensors)

259

@@ -282,14 +280,6 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):

280 sampling_metadata)

281 return logits

282

285 - def sample(

286 - self,

287 - logits: torch.Tensor,

288 - sampling_metadata: SamplingMetadata,

289 - ) -> Optional[SamplerOutput]:

290 - next_tokens = self.sampler(logits, sampling_metadata)

291 - return next_tokens

292 -

283 def load_weights(self, weights: Iterable[Tuple[str,

284 torch.Tensor]]) -> Set[str]:

285 params_dict = dict(self.named_parameters(remove_duplicate=False))

vllm/model_executor/models/gpt_bigcode.py

-10

@@ -35,7 +35,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

35 RowParallelLinear)

36 from vllm.model_executor.layers.logits_processor import LogitsProcessor

37 from vllm.model_executor.layers.quantization import QuantizationConfig

38 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

38 from vllm.model_executor.layers.vocab_parallel_embedding import (

39 ParallelLMHead, VocabParallelEmbedding)

40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -302,7 +301,6 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

301 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size

302 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

303 config.vocab_size)

305 - self.sampler = get_sampler()

304 self.make_empty_intermediate_tensors = (

305 self.transformer.make_empty_intermediate_tensors)

306

@@ -329,14 +327,6 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

327 sampling_metadata)

328 return logits

329

332 - def sample(

333 - self,

334 - logits: torch.Tensor,

335 - sampling_metadata: SamplingMetadata,

336 - ) -> Optional[SamplerOutput]:

337 - next_tokens = self.sampler(logits, sampling_metadata)

338 - return next_tokens

339 -

330 def load_weights(self, weights: Iterable[Tuple[str,

331 torch.Tensor]]) -> Set[str]:

332 loader = AutoWeightsLoader(

vllm/model_executor/models/gpt_j.py

-10

@@ -34,7 +34,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

34 from vllm.model_executor.layers.logits_processor import LogitsProcessor

35 from vllm.model_executor.layers.quantization import QuantizationConfig

36 from vllm.model_executor.layers.rotary_embedding import get_rope

37 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

37 from vllm.model_executor.layers.vocab_parallel_embedding import (

38 ParallelLMHead, VocabParallelEmbedding)

39 from vllm.model_executor.model_loader.weight_utils import (

@@ -306,7 +305,6 @@ class GPTJForCausalLM(nn.Module, SupportsPP):

305 quant_config=quant_config,

306 )

307 self.logits_processor = LogitsProcessor(config.vocab_size)

309 - self.sampler = get_sampler()

308 self.make_empty_intermediate_tensors = (

309 self.transformer.make_empty_intermediate_tensors)

310

@@ -333,14 +331,6 @@ class GPTJForCausalLM(nn.Module, SupportsPP):

331 sampling_metadata, self.lm_head.bias)

332 return logits

333

336 - def sample(

337 - self,

338 - logits: torch.Tensor,

339 - sampling_metadata: SamplingMetadata,

340 - ) -> Optional[SamplerOutput]:

341 - next_tokens = self.sampler(logits, sampling_metadata)

342 - return next_tokens

343 -

334 def load_weights(self, weights: Iterable[Tuple[str,

335 torch.Tensor]]) -> Set[str]:

336 loader = AutoWeightsLoader(self)

vllm/model_executor/models/gpt_neox.py

-10

@@ -34,7 +34,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

34 from vllm.model_executor.layers.logits_processor import LogitsProcessor

35 from vllm.model_executor.layers.quantization import QuantizationConfig

36 from vllm.model_executor.layers.rotary_embedding import get_rope

37 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

37 from vllm.model_executor.layers.vocab_parallel_embedding import (

38 ParallelLMHead, VocabParallelEmbedding)

39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -299,7 +298,6 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):

298 if self.config.tie_word_embeddings:

299 self.embed_out.weight = self.gpt_neox.embed_in.weight

300 self.logits_processor = LogitsProcessor(config.vocab_size)

302 - self.sampler = get_sampler()

301 self.make_empty_intermediate_tensors = (

302 self.gpt_neox.make_empty_intermediate_tensors)

303

@@ -326,14 +324,6 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):

324 sampling_metadata)

325 return logits

326

329 - def sample(

330 - self,

331 - logits: torch.Tensor,

332 - sampling_metadata: SamplingMetadata,

333 - ) -> Optional[SamplerOutput]:

334 - next_tokens = self.sampler(logits, sampling_metadata)

335 - return next_tokens

336 -

327 def load_weights(self, weights: Iterable[Tuple[str,

328 torch.Tensor]]) -> Set[str]:

329 loader = AutoWeightsLoader(self)

vllm/model_executor/models/granite.py

-11

@@ -41,7 +41,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

41 from vllm.model_executor.layers.quantization.base_config import (

42 QuantizationConfig)

43 from vllm.model_executor.layers.rotary_embedding import get_rope

44 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

44 from vllm.model_executor.layers.vocab_parallel_embedding import (

45 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

46 from vllm.model_executor.model_loader.weight_utils import (

@@ -441,8 +440,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

440 else:

441 self.lm_head = PPMissingLayer()

442

444 - self.sampler = get_sampler()

445 -

443 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:

444 return self.model.get_input_embeddings(input_ids)

445

@@ -464,14 +461,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

461 sampling_metadata)

462 return logits

463

467 - def sample(

468 - self,

469 - logits: torch.Tensor,

470 - sampling_metadata: SamplingMetadata,

471 - ) -> Optional[SamplerOutput]:

472 - next_tokens = self.sampler(logits, sampling_metadata)

473 - return next_tokens

474 -

464 def make_empty_intermediate_tensors(

465 self, batch_size: int, dtype: torch.dtype,

466 device: torch.device) -> IntermediateTensors:

vllm/model_executor/models/granitemoe.py

-11

@@ -41,7 +41,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

41 from vllm.model_executor.layers.quantization.base_config import (

42 QuantizationConfig)

43 from vllm.model_executor.layers.rotary_embedding import get_rope

44 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

44 from vllm.model_executor.layers.vocab_parallel_embedding import (

45 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

46 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -391,8 +390,6 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

390 scale=1 /

391 self.config.logits_scaling)

392

394 - self.sampler = get_sampler()

395 -

393 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:

394 return self.model.get_input_embeddings(input_ids)

395

@@ -428,14 +425,6 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

425 device=device),

426 })

427

431 - def sample(

432 - self,

433 - logits: Optional[torch.Tensor],

434 - sampling_metadata: SamplingMetadata,

435 - ) -> Optional[SamplerOutput]:

436 - next_tokens = self.sampler(logits, sampling_metadata)

437 - return next_tokens

438 -

428 def load_weights(self, weights: Iterable[Tuple[str,

429 torch.Tensor]]) -> Set[str]:

430 loader = AutoWeightsLoader(

vllm/model_executor/models/granitemoeshared.py

-11

@@ -20,7 +20,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

20 from vllm.model_executor.layers.logits_processor import LogitsProcessor

21 from vllm.model_executor.layers.quantization.base_config import (

22 QuantizationConfig)

23 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

23 from vllm.model_executor.layers.vocab_parallel_embedding import (

24 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

25 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -295,8 +294,6 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

294 scale=1 /

295 self.config.logits_scaling)

296

298 - self.sampler = get_sampler()

299 -

297 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:

298 return self.model.get_input_embeddings(input_ids)

299

@@ -332,14 +329,6 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

329 device=device),

330 })

331

335 - def sample(

336 - self,

337 - logits: Optional[torch.Tensor],

338 - sampling_metadata: SamplingMetadata,

339 - ) -> Optional[SamplerOutput]:

340 - next_tokens = self.sampler(logits, sampling_metadata)

341 - return next_tokens

342 -

332 def load_weights(self, weights: Iterable[Tuple[str,

333 torch.Tensor]]) -> Set[str]:

334 loader = AutoWeightsLoader(

vllm/model_executor/models/grok1.py

-10

@@ -39,7 +39,6 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

39 from vllm.model_executor.layers.logits_processor import LogitsProcessor

40 from vllm.model_executor.layers.quantization import QuantizationConfig

41 from vllm.model_executor.layers.rotary_embedding import get_rope

42 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

42 from vllm.model_executor.layers.vocab_parallel_embedding import (

43 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

44 from vllm.model_executor.model_loader.weight_utils import (

@@ -521,7 +520,6 @@ class Grok1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

520 config.vocab_size,

521 self.output_multiplier_scale)

522

524 - self.sampler = get_sampler()

523 self.make_empty_intermediate_tensors = (

524 self.model.make_empty_intermediate_tensors)

525

@@ -551,14 +549,6 @@ class Grok1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

549 sampling_metadata)

550 return logits

551

554 - def sample(

555 - self,

556 - logits: Optional[torch.Tensor],

557 - sampling_metadata: SamplingMetadata,

558 - ) -> Optional[SamplerOutput]:

559 - next_tokens = self.sampler(logits, sampling_metadata)

560 - return next_tokens

561 -

552 def load_weights(self, weights: Iterable[Tuple[str,

553 torch.Tensor]]) -> Set[str]:

554 skip_prefixes = ["rotary_emb.inv_freq"]

vllm/model_executor/models/idefics3.py

-10

@@ -28,7 +28,6 @@ from vllm.config import VllmConfig

28 from vllm.model_executor.layers.linear import ReplicatedLinear

29 from vllm.model_executor.layers.logits_processor import LogitsProcessor

30 from vllm.model_executor.layers.quantization import QuantizationConfig

31 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

31 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

32 from vllm.model_executor.models.module_mapping import MultiModelKeys

33 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -603,7 +602,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

602 if self.config.text_config.tie_word_embeddings:

603 self.lm_head.weight = self.model.text_model.wte.weight

604 self.logits_processor = LogitsProcessor(config.text_config.vocab_size)

606 - self.sampler = get_sampler()

605

606 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:

607 h = w = self.config.vision_config.image_size

@@ -754,14 +752,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

752 sampling_metadata)

753 return logits

754

757 - def sample(

758 - self,

759 - logits: torch.Tensor,

760 - sampling_metadata: SamplingMetadata,

761 - ) -> Optional[SamplerOutput]:

762 - next_tokens = self.sampler(logits, sampling_metadata)

763 - return next_tokens

764 -

755 def load_weights(self, weights: Iterable[Tuple[str,

756 torch.Tensor]]) -> Set[str]:

757 loader = AutoWeightsLoader(self)

vllm/model_executor/models/interfaces_base.py

-9

@@ -13,7 +13,6 @@ from vllm.utils import supports_kw

13 if TYPE_CHECKING:

14 from vllm.config import VllmConfig

15 from vllm.model_executor.layers.pooler import PoolerOutput

16 - from vllm.model_executor.layers.sampler import SamplerOutput

16 from vllm.model_executor.pooling_metadata import PoolingMetadata

17 from vllm.model_executor.sampling_metadata import SamplingMetadata

18

@@ -103,14 +102,6 @@ class VllmModelForTextGeneration(VllmModel[T], Protocol[T]):

102 """Return `None` if TP rank > 0."""

103 ...

104

106 - def sample(

107 - self,

108 - logits: T,

109 - sampling_metadata: "SamplingMetadata",

110 - ) -> "SamplerOutput":

111 - """Only called on TP rank 0."""

112 - ...

113 -

105

106 @overload

107 def is_text_generation_model(

vllm/model_executor/models/internlm2.py

+1 -11

@@ -23,7 +23,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

23 from vllm.model_executor.layers.pooler import Pooler, PoolingType

24 from vllm.model_executor.layers.quantization import QuantizationConfig

25 from vllm.model_executor.layers.rotary_embedding import get_rope

26 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

26 from vllm.model_executor.layers.vocab_parallel_embedding import (

27 ParallelLMHead, VocabParallelEmbedding)

28 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -336,7 +335,6 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):

335 if self.config.tie_word_embeddings:

336 self.output.weight = self.model.tok_embeddings.weight

337 self.logits_processor = LogitsProcessor(config.vocab_size)

339 - self.sampler = get_sampler()

338 self.make_empty_intermediate_tensors = (

339 self.model.make_empty_intermediate_tensors)

340

@@ -363,14 +361,6 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):

361 sampling_metadata)

362 return logits

363

366 - def sample(

367 - self,

368 - logits: torch.Tensor,

369 - sampling_metadata: SamplingMetadata,

370 - ) -> Optional[SamplerOutput]:

371 - next_tokens = self.sampler(logits, sampling_metadata)

372 - return next_tokens

373 -

364 def load_weights(self, weights: Iterable[Tuple[str,

365 torch.Tensor]]) -> Set[str]:

366 stacked_params_mapping = [

@@ -423,7 +413,7 @@ class InternLM2ForRewardModel(InternLM2ForCausalLM):

413 prefix=prefix,

414 model_type=model_type)

415

426 - for attr in ("output", "logits_processor", "sampler"):

416 + for attr in ("output", "logits_processor"):

417 delattr(self, attr)

418

419 config = vllm_config.model_config.hf_config

vllm/model_executor/models/internvl.py

+1 -17

@@ -8,7 +8,6 @@

8 # --------------------------------------------------------

9 from abc import ABC, abstractmethod

10 from collections.abc import Iterable, Mapping, Sequence

11 -from functools import cached_property

11 from typing import Literal, Optional, Set, Tuple, TypedDict, TypeVar, Union

12

13 import torch

@@ -20,7 +19,6 @@ from transformers import BatchEncoding, PretrainedConfig, TensorType

19 from vllm.config import VllmConfig

20 from vllm.model_executor.layers.quantization import QuantizationConfig

21 from vllm.model_executor.layers.quantization.awq import AWQConfig

23 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

22 from vllm.model_executor.models.intern_vit import (InternVisionModel,

23 InternVisionPatchModel)

24 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -698,13 +696,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

696 (llm_quant_config is not None):

697 quant_config.modules_to_not_convert.append("vision_model")

698

701 - @cached_property

702 - def sampler(self):

703 - if hasattr(self.language_model, "sampler"):

704 - return self.language_model.sampler

705 -

706 - return get_sampler()

707 -

699 def _init_vision_model(

700 self,

701 config: PretrainedConfig,

@@ -903,7 +894,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

894 intermediate_tensors: Optional[IntermediateTensors] = None,

895 inputs_embeds: Optional[torch.Tensor] = None,

896 **kwargs: object,

906 - ) -> Union[SamplerOutput, IntermediateTensors]:

897 + ) -> IntermediateTensors:

898

899 if intermediate_tensors is not None:

900 input_ids = None

@@ -941,13 +932,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

932 return self.language_model.compute_logits(hidden_states,

933 sampling_metadata)

934

944 - def sample(

945 - self,

946 - logits: torch.Tensor,

947 - sampling_metadata: SamplingMetadata,

948 - ) -> Optional[SamplerOutput]:

949 - return self.language_model.sample(logits, sampling_metadata)

950 -

935 def load_weights(self, weights: Iterable[Tuple[str,

936 torch.Tensor]]) -> Set[str]:

937 # unused modules appear in OpenGVLab/InternVideo2_5_Chat_8B

vllm/model_executor/models/jais.py

-10

@@ -36,7 +36,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

36 RowParallelLinear)

37 from vllm.model_executor.layers.logits_processor import LogitsProcessor

38 from vllm.model_executor.layers.quantization import QuantizationConfig

39 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

39 from vllm.model_executor.layers.vocab_parallel_embedding import (

40 ParallelLMHead, VocabParallelEmbedding)

41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -308,7 +307,6 @@ class JAISLMHeadModel(nn.Module, SupportsPP):

307 config.mup_width_scale)

308 self.logits_processor = LogitsProcessor(vocab_size=config.vocab_size,

309 scale=self.output_logits_scale)

311 - self.sampler = get_sampler()

310 self.make_empty_intermediate_tensors = (

311 self.transformer.make_empty_intermediate_tensors)

312

@@ -335,14 +333,6 @@ class JAISLMHeadModel(nn.Module, SupportsPP):

333 sampling_metadata)

334 return logits

335

338 - def sample(

339 - self,

340 - logits: torch.Tensor,

341 - sampling_metadata: SamplingMetadata,

342 - ) -> Optional[SamplerOutput]:

343 - next_tokens = self.sampler(logits, sampling_metadata)

344 - return next_tokens

345 -

336 def load_weights(self, weights: Iterable[Tuple[str,

337 torch.Tensor]]) -> Set[str]:

338 params_dict = dict(self.named_parameters(remove_duplicate=False))

vllm/model_executor/models/jamba.py

-10

@@ -19,7 +19,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

19 from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer

20 from vllm.model_executor.layers.pooler import Pooler, PoolingType

21 from vllm.model_executor.layers.quantization import QuantizationConfig

22 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

22 from vllm.model_executor.layers.vocab_parallel_embedding import (

23 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

24 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -409,7 +408,6 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,

408

409 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

410 config.vocab_size)

412 - self.sampler = get_sampler()

411

412 self.make_empty_intermediate_tensors = (

413 self.model.make_empty_intermediate_tensors)

@@ -466,14 +464,6 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,

464 sampling_metadata)

465 return logits

466

469 - def sample(

470 - self,

471 - logits: Optional[torch.Tensor],

472 - sampling_metadata: SamplingMetadata,

473 - ) -> Optional[SamplerOutput]:

474 - next_tokens = self.sampler(logits, sampling_metadata)

475 - return next_tokens

476 -

467 def load_weights(self, weights: Iterable[Tuple[str,

468 torch.Tensor]]) -> Set[str]:

469 stacked_params_mapping = [

vllm/model_executor/models/kimi_vl.py

+1 -11

@@ -58,7 +58,6 @@ from vllm.distributed import (get_tensor_model_parallel_rank,

58 get_tensor_model_parallel_world_size)

59 from vllm.model_executor.layers.fused_moe import FusedMoE

60 from vllm.model_executor.layers.logits_processor import LogitsProcessor

61 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

61 from vllm.model_executor.layers.vocab_parallel_embedding import (

62 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)

63 from vllm.model_executor.model_loader.weight_utils import (

@@ -298,7 +297,6 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal):

297 logit_scale = getattr(config, "logit_scale", 1.0)

298 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

299 config.vocab_size, logit_scale)

301 - self.sampler = get_sampler()

300 self.media_placeholder: int = self.config.media_placeholder_token_id

301 self.tp_rank = get_tensor_model_parallel_rank()

302 self.tp_world_size = get_tensor_model_parallel_world_size()

@@ -409,7 +407,7 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal):

407 intermediate_tensors: Optional[IntermediateTensors] = None,

408 inputs_embeds: Optional[torch.Tensor] = None,

409 **kwargs: object,

412 - ) -> SamplerOutput:

410 + ) -> IntermediateTensors:

411 if intermediate_tensors is not None:

412 inputs_embeds = None

413 # NOTE: In v1, inputs_embeds is always generated at model runner from

@@ -447,14 +445,6 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal):

445 sampling_metadata, **kwargs)

446 return logits

447

450 - def sample(

451 - self,

452 - logits: torch.Tensor,

453 - sampling_metadata: SamplingMetadata,

454 - ) -> Optional[SamplerOutput]:

455 - next_tokens = self.sampler(logits, sampling_metadata)

456 - return next_tokens

457 -

448 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

449 config = self.config.text_config

450 _KEYS_TO_MODIFY_MAPPING = {

vllm/model_executor/models/llama.py

-8

@@ -40,7 +40,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

40 from vllm.model_executor.layers.logits_processor import LogitsProcessor

41 from vllm.model_executor.layers.quantization import QuantizationConfig

42 from vllm.model_executor.layers.rotary_embedding import get_rope

43 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

43 from vllm.model_executor.layers.vocab_parallel_embedding import (

44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

45 from vllm.model_executor.model_loader.weight_utils import (

@@ -515,8 +514,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

514 else:

515 self.lm_head = PPMissingLayer()

516

518 - self.sampler = get_sampler()

519 -

517 self.make_empty_intermediate_tensors = (

518 self.model.make_empty_intermediate_tensors)

519

@@ -551,11 +548,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

548 sampling_metadata)

549 return logits

550

554 - def sample(self, logits: torch.Tensor,

555 - sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:

556 - next_tokens = self.sampler(logits, sampling_metadata)

557 - return next_tokens

558 -

551 def load_weights(self, weights: Iterable[Tuple[str,

552 torch.Tensor]]) -> Set[str]:

553 loader = AutoWeightsLoader(

vllm/model_executor/models/llava.py

-16

@@ -2,7 +2,6 @@

2

3 from abc import abstractmethod

4 from collections.abc import Iterable, Mapping, Sequence

5 -from functools import cached_property

5 from typing import (Final, Literal, Optional, Protocol, Set, Tuple, TypedDict,

6 TypeVar, Union, cast)

7

@@ -23,7 +22,6 @@ from vllm.model_executor.layers.activation import get_act_fn

22 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

23 RowParallelLinear)

24 from vllm.model_executor.layers.quantization import QuantizationConfig

26 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

25 from vllm.model_executor.sampling_metadata import SamplingMetadata

26 from vllm.multimodal import MULTIMODAL_REGISTRY

27 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -546,13 +544,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

544 self.make_empty_intermediate_tensors = (

545 self.language_model.make_empty_intermediate_tensors)

546

549 - @cached_property

550 - def sampler(self):

551 - if hasattr(self.language_model, "sampler"):

552 - return self.language_model.sampler

553 -

554 - return get_sampler()

555 -

547 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:

548 h = w = self.config.vision_config.image_size

549 expected_dims = (3, h, w)

@@ -763,13 +754,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

754 return self.language_model.compute_logits(hidden_states,

755 sampling_metadata)

756

766 - def sample(

767 - self,

768 - logits: torch.Tensor,

769 - sampling_metadata: SamplingMetadata,

770 - ) -> Optional[SamplerOutput]:

771 - return self.language_model.sample(logits, sampling_metadata)

772 -

757 def load_weights(self, weights: Iterable[Tuple[str,

758 torch.Tensor]]) -> Set[str]:

759 loader = AutoWeightsLoader(self)

vllm/model_executor/models/llava_next.py

-16

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from abc import abstractmethod

4 -from functools import cached_property

4 from typing import (Final, Iterable, List, Literal, Mapping, Optional,

5 Protocol, Set, Tuple, TypedDict, TypeVar, Union)

6

@@ -13,7 +12,6 @@ from transformers.models.llava_next.modeling_llava_next import (

12 from typing_extensions import NotRequired

13

14 from vllm.config import VllmConfig

16 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

15 from vllm.model_executor.sampling_metadata import SamplingMetadata

16 from vllm.multimodal import MULTIMODAL_REGISTRY

17 from vllm.multimodal.inputs import MultiModalFieldConfig

@@ -250,13 +248,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

248 self.make_empty_intermediate_tensors = (

249 self.language_model.make_empty_intermediate_tensors)

250

253 - @cached_property

254 - def sampler(self):

255 - if hasattr(self.language_model, "sampler"):

256 - return self.language_model.sampler

257 -

258 - return get_sampler()

259 -

251 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:

252 expected_dims = (2, )

253

@@ -585,13 +576,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

576 return self.language_model.compute_logits(hidden_states,

577 sampling_metadata)

578

588 - def sample(

589 - self,

590 - logits: torch.Tensor,

591 - sampling_metadata: SamplingMetadata,

592 - ) -> Optional[SamplerOutput]:

593 - return self.language_model.sample(logits, sampling_metadata)

594 -

579 def load_weights(self, weights: Iterable[Tuple[str,

580 torch.Tensor]]) -> Set[str]:

581 loader = AutoWeightsLoader(self)

vllm/model_executor/models/llava_next_video.py

-16

@@ -2,7 +2,6 @@

2

3 import math

4 from collections.abc import Iterable, Mapping, Sequence

5 -from functools import cached_property

5 from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union

6

7 import torch

@@ -12,7 +11,6 @@ from transformers import (BatchFeature, LlavaNextVideoConfig,

11

12 from vllm.config import VllmConfig

13 from vllm.model_executor.layers.activation import get_act_fn

15 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

14 from vllm.model_executor.models.clip import CLIPVisionModel

15 from vllm.model_executor.sampling_metadata import SamplingMetadata

16 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -301,13 +299,6 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

299 self.make_empty_intermediate_tensors = (

300 self.language_model.model.make_empty_intermediate_tensors)

301

304 - @cached_property

305 - def sampler(self):

306 - if hasattr(self.language_model, "sampler"):

307 - return self.language_model.sampler

308 -

309 - return get_sampler()

310 -

302 def _validate_video_pixel_values(

303 self, data: Union[torch.Tensor, List[torch.Tensor]]

304 ) -> Union[torch.Tensor, List[torch.Tensor]]:

@@ -469,13 +460,6 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

460 return self.language_model.compute_logits(hidden_states,

461 sampling_metadata)

462

472 - def sample(

473 - self,

474 - logits: torch.Tensor,

475 - sampling_metadata: SamplingMetadata,

476 - ) -> Optional[SamplerOutput]:

477 - return self.language_model.sample(logits, sampling_metadata)

478 -

463 def load_weights(self, weights: Iterable[Tuple[str,

464 torch.Tensor]]) -> Set[str]:

465 loader = AutoWeightsLoader(

vllm/model_executor/models/llava_onevision.py

-16

@@ -2,7 +2,6 @@

2

3 import math

4 from collections.abc import Iterable, Mapping, Sequence

5 -from functools import cached_property

5 from typing import (Final, List, Literal, Optional, Protocol, Set, Tuple,

6 TypedDict, Union)

7

@@ -16,7 +15,6 @@ from typing_extensions import NotRequired

15

16 from vllm.config import VllmConfig

17 from vllm.model_executor.layers.activation import get_act_fn

19 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

18 from vllm.model_executor.sampling_metadata import SamplingMetadata

19 from vllm.multimodal import MULTIMODAL_REGISTRY

20 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -455,13 +453,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

453 self.make_empty_intermediate_tensors = (

454 self.language_model.model.make_empty_intermediate_tensors)

455

458 - @cached_property

459 - def sampler(self):

460 - if hasattr(self.language_model, "sampler"):

461 - return self.language_model.sampler

462 -

463 - return get_sampler()

464 -

456 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:

457 expected_dims = (2, )

458

@@ -957,13 +948,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

948 return self.language_model.compute_logits(hidden_states,

949 sampling_metadata)

950

960 - def sample(

961 - self,

962 - logits: torch.Tensor,

963 - sampling_metadata: SamplingMetadata,

964 - ) -> Optional[SamplerOutput]:

965 - return self.language_model.sample(logits, sampling_metadata)

966 -

951 def load_weights(self, weights: Iterable[Tuple[str,

952 torch.Tensor]]) -> Set[str]:

953 loader = AutoWeightsLoader(self)

vllm/model_executor/models/mamba.py

-10

@@ -14,7 +14,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

14 from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer

15 from vllm.model_executor.layers.quantization.base_config import (

16 QuantizationConfig)

17 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

17 from vllm.model_executor.layers.vocab_parallel_embedding import (

18 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

19 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -213,7 +212,6 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP,

212

213 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

214 config.vocab_size)

216 - self.sampler = get_sampler()

215

216 self.make_empty_intermediate_tensors = (

217 self.backbone.make_empty_intermediate_tensors)

@@ -267,14 +265,6 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP,

265 sampling_metadata)

266 return logits

267

270 - def sample(

271 - self,

272 - logits: Optional[torch.Tensor],

273 - sampling_metadata: SamplingMetadata,

274 - ) -> Optional[SamplerOutput]:

275 - next_tokens = self.sampler(logits, sampling_metadata)

276 - return next_tokens

277 -

268 def load_weights(self, weights: Iterable[Tuple[str,

269 torch.Tensor]]) -> Set[str]:

270 loader = AutoWeightsLoader(self)

vllm/model_executor/models/mamba2.py

-10

@@ -19,7 +19,6 @@ from vllm.model_executor.layers.mamba.mamba_mixer2 import (

19 MambaMixer2, extra_groups_for_head_shards)

20 from vllm.model_executor.layers.quantization.base_config import (

21 QuantizationConfig)

22 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

22 from vllm.model_executor.layers.vocab_parallel_embedding import (

23 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

24 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -208,7 +207,6 @@ class Mamba2ForCausalLM(nn.Module, HasInnerState, IsAttentionFree,

207

208 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

209 config.vocab_size)

211 - self.sampler = get_sampler()

210

211 self.make_empty_intermediate_tensors = (

212 self.backbone.make_empty_intermediate_tensors)

@@ -282,14 +280,6 @@ class Mamba2ForCausalLM(nn.Module, HasInnerState, IsAttentionFree,

280 sampling_metadata)

281 return logits

282

285 - def sample(

286 - self,

287 - logits: Optional[torch.Tensor],

288 - sampling_metadata: SamplingMetadata,

289 - ) -> Optional[SamplerOutput]:

290 - next_tokens = self.sampler(logits, sampling_metadata)

291 - return next_tokens

292 -

283 def load_weights(self, weights: Iterable[Tuple[str,

284 torch.Tensor]]) -> Set[str]:

285 params_dict = dict(self.named_parameters())

vllm/model_executor/models/minicpm.py

-10

@@ -45,7 +45,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

45 from vllm.model_executor.layers.logits_processor import LogitsProcessor

46 from vllm.model_executor.layers.quantization import QuantizationConfig

47 from vllm.model_executor.layers.rotary_embedding import get_rope

48 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

48 from vllm.model_executor.layers.vocab_parallel_embedding import (

49 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -553,7 +552,6 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

552

553 self.logits_processor = LogitsProcessor(unpadded_vocab_size,

554 config.vocab_size)

556 - self.sampler = get_sampler()

555 self.make_empty_intermediate_tensors = (

556 self.model.make_empty_intermediate_tensors)

557

@@ -584,14 +582,6 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

582 sampling_metadata)

583 return logits

584

587 - def sample(

588 - self,

589 - logits: torch.Tensor,

590 - sampling_metadata: SamplingMetadata,

591 - ) -> Optional[SamplerOutput]:

592 - next_tokens = self.sampler(logits, sampling_metadata)

593 - return next_tokens

594 -

585 def load_weights(self, weights: Iterable[Tuple[str,

586 torch.Tensor]]) -> Set[str]:

587 loader = AutoWeightsLoader(

vllm/model_executor/models/minicpmv.py

+1 -17

@@ -25,7 +25,7 @@

25 import math

26 from collections import defaultdict

27 from collections.abc import Iterable, Mapping, Sequence

28 -from functools import cached_property, partial

28 +from functools import partial

29 from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict,

30 Union)

31

@@ -40,7 +40,6 @@ from vllm.config import VllmConfig

40 from vllm.model_executor.layers.quantization import QuantizationConfig

41 from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,

42 get_2d_sincos_pos_embed)

43 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

43 from vllm.model_executor.model_loader.utils import set_default_torch_dtype

44 from vllm.model_executor.models.llama import LlamaForCausalLM

45 from vllm.model_executor.models.minicpm import MiniCPMForCausalLM

@@ -758,13 +757,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

757 self.make_empty_intermediate_tensors = (

758 self.llm.make_empty_intermediate_tensors)

759

761 - @cached_property

762 - def sampler(self):

763 - if hasattr(self.llm, "sampler"):

764 - return self.llm.sampler

765 -

766 - return get_sampler()

767 -

760 def _parse_and_validate_vision_input(

761 self,

762 modality: str,

@@ -946,14 +938,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

938 ) -> Optional[torch.Tensor]:

939 return self.llm.compute_logits(hidden_states, sampling_metadata)

940

949 - def sample(

950 - self,

951 - logits: torch.Tensor,

952 - sampling_metadata: SamplingMetadata,

953 - ) -> Optional[SamplerOutput]:

954 - next_tokens = self.sampler(logits, sampling_metadata)

955 - return next_tokens

956 -

941 def load_weights(self, weights: Iterable[Tuple[str,

942 torch.Tensor]]) -> Set[str]:

943 loader = AutoWeightsLoader(self)

vllm/model_executor/models/minimax_text_01.py

-12

@@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

33 from vllm.model_executor.layers.logits_processor import LogitsProcessor

34 from vllm.model_executor.layers.quantization.base_config import (

35 QuantizationConfig)

36 -from vllm.model_executor.layers.sampler import Sampler

36 from vllm.model_executor.layers.vocab_parallel_embedding import (

37 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -994,7 +993,6 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,

993 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

994 self.config.vocab_size)

995

997 - self.sampler = Sampler()

996 else:

997 self.lm_head = PPMissingLayer()

998

@@ -1030,16 +1028,6 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,

1028

1029 return logits

1030

1033 - def sample(

1034 - self,

1035 - logits: torch.Tensor,

1036 - sampling_metadata: SamplingMetadata,

1037 - ):

1038 -

1039 - next_tokens = self.sampler(logits, sampling_metadata)

1040 -

1041 - return next_tokens

1042 -

1031 def make_empty_intermediate_tensors(

1032 self, batch_size: int, dtype: torch.dtype,

1033 device: torch.device) -> IntermediateTensors:

vllm/model_executor/models/mistral3.py

-16

@@ -2,7 +2,6 @@

2

3 from abc import abstractmethod

4 from collections.abc import Iterable, Mapping, Sequence

5 -from functools import cached_property

5 from typing import (Final, Literal, Optional, Protocol, Set, Tuple, TypedDict,

6 TypeVar, Union)

7

@@ -19,7 +18,6 @@ from vllm.model_executor.layers.layernorm import RMSNorm

18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

19 RowParallelLinear)

20 from vllm.model_executor.layers.quantization import QuantizationConfig

22 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

21 from vllm.model_executor.sampling_metadata import SamplingMetadata

22 from vllm.multimodal import MULTIMODAL_REGISTRY

23 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -435,13 +433,6 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,

433 self.make_empty_intermediate_tensors = (

434 self.language_model.make_empty_intermediate_tensors)

435

438 - @cached_property

439 - def sampler(self):

440 - if hasattr(self.language_model, "sampler"):

441 - return self.language_model.sampler

442 -

443 - return get_sampler()

444 -

436 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:

437 h = w = self.config.vision_config.image_size

438 expected_dims = (3, h, w)

@@ -598,13 +589,6 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,

589 return self.language_model.compute_logits(hidden_states,

590 sampling_metadata)

591

601 - def sample(

602 - self,

603 - logits: torch.Tensor,

604 - sampling_metadata: SamplingMetadata,

605 - ) -> Optional[SamplerOutput]:

606 - return self.language_model.sample(logits, sampling_metadata)

607 -

592 def load_weights(self, weights: Iterable[Tuple[str,

593 torch.Tensor]]) -> Set[str]:

594 loader = AutoWeightsLoader(self)

vllm/model_executor/models/mixtral.py

-10

@@ -40,7 +40,6 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

40 from vllm.model_executor.layers.logits_processor import LogitsProcessor

41 from vllm.model_executor.layers.quantization import QuantizationConfig

42 from vllm.model_executor.layers.rotary_embedding import get_rope

43 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

43 from vllm.model_executor.layers.vocab_parallel_embedding import (

44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

45 from vllm.model_executor.model_loader.weight_utils import (

@@ -454,7 +453,6 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

453 self.lm_head.weight = self.model.embed_tokens.weight

454 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

455 config.vocab_size)

457 - self.sampler = get_sampler()

456 self.make_empty_intermediate_tensors = (

457 self.model.make_empty_intermediate_tensors)

458

@@ -481,14 +479,6 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

479 sampling_metadata)

480 return logits

481

484 - def sample(

485 - self,

486 - logits: Optional[torch.Tensor],

487 - sampling_metadata: SamplingMetadata,

488 - ) -> Optional[SamplerOutput]:

489 - next_tokens = self.sampler(logits, sampling_metadata)

490 - return next_tokens

491 -

482 def load_weights(self, weights: Iterable[Tuple[str,

483 torch.Tensor]]) -> Set[str]:

484 loader = AutoWeightsLoader(self, skip_prefixes=["rotary_emb.inv_freq"])

vllm/model_executor/models/mixtral_quant.py

-10

@@ -42,7 +42,6 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

42 from vllm.model_executor.layers.logits_processor import LogitsProcessor

43 from vllm.model_executor.layers.quantization import QuantizationConfig

44 from vllm.model_executor.layers.rotary_embedding import get_rope

45 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

45 from vllm.model_executor.layers.vocab_parallel_embedding import (

46 ParallelLMHead, VocabParallelEmbedding)

47 from vllm.model_executor.model_loader.weight_utils import (

@@ -372,7 +371,6 @@ class MixtralForCausalLM(nn.Module, SupportsPP):

371 if self.config.tie_word_embeddings:

372 self.lm_head.weight = self.model.embed_tokens.weight

373 self.logits_processor = LogitsProcessor(config.vocab_size)

375 - self.sampler = get_sampler()

374 self.make_empty_intermediate_tensors = (

375 self.model.make_empty_intermediate_tensors)

376

@@ -399,14 +397,6 @@ class MixtralForCausalLM(nn.Module, SupportsPP):

397 sampling_metadata)

398 return logits

399

402 - def sample(

403 - self,

404 - logits: Optional[torch.Tensor],

405 - sampling_metadata: SamplingMetadata,

406 - ) -> Optional[SamplerOutput]:

407 - next_tokens = self.sampler(logits, sampling_metadata)

408 - return next_tokens

409 -

400 def load_weights(self, weights: Iterable[Tuple[str,

401 torch.Tensor]]) -> Set[str]:

402 stacked_params_mapping = [

vllm/model_executor/models/mllama.py

-10

@@ -47,7 +47,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

47 RowParallelLinear)

48 from vllm.model_executor.layers.logits_processor import LogitsProcessor

49 from vllm.model_executor.layers.quantization import QuantizationConfig

50 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

50 from vllm.model_executor.layers.vocab_parallel_embedding import (

51 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

52 from vllm.model_executor.model_loader.weight_utils import (

@@ -1211,7 +1210,6 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,

1210 )

1211 self.logits_processor = LogitsProcessor(config.output_hidden_states,

1212 config.text_config.vocab_size)

1214 - self.sampler = get_sampler()

1213

1214 def compute_logits(

1215 self,

@@ -1222,14 +1220,6 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,

1220 hidden_states, sampling_metadata)

1221 return logits

1222

1225 - def sample(

1226 - self,

1227 - logits: torch.Tensor,

1228 - sampling_metadata: SamplingMetadata,

1229 - ) -> Optional[SamplerOutput]:

1230 - next_tokens = self.sampler(logits, sampling_metadata)

1231 - return next_tokens

1232 -

1223 def unpack_data(self,

1224 image_data: Union[List[torch.Tensor], torch.Tensor],

1225 padding_value=0) -> torch.Tensor:

vllm/model_executor/models/mllama4.py

-13

@@ -17,7 +17,6 @@

17 # limitations under the License.

18 import math

19 from collections.abc import Iterable, Mapping

20 -from functools import cached_property

20 from itertools import tee

21 from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union

22

@@ -38,7 +37,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

37 RowParallelLinear)

38 from vllm.model_executor.layers.quantization import QuantizationConfig

39 from vllm.model_executor.layers.rotary_embedding import get_rope

41 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

40 from vllm.model_executor.model_loader.loader import _initialize_model

41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

42 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -682,13 +680,6 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

680 self.make_empty_intermediate_tensors = (

681 self.language_model.make_empty_intermediate_tensors)

682

685 - @cached_property

686 - def sampler(self):

687 - if hasattr(self.language_model, "sampler"):

688 - return self.language_model.sampler

689 -

690 - return get_sampler()

691 -

683 def _parse_and_validate_image_input(

684 self, **kwargs: object) -> Optional[Llama4ImagePatchInputs]:

685 # num_images, 1, num_chunks, channel, image_size, image_size

@@ -785,10 +776,6 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

776 return self.language_model.compute_logits(hidden_states,

777 sampling_metadata)

778

788 - def sample(self, logits: torch.Tensor,

789 - sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:

790 - return self.language_model.sample(logits, sampling_metadata)

791 -

779 def separate_weights(

780 self,

781 weights: Iterable[Tuple[str, torch.Tensor]],

vllm/model_executor/models/molmo.py

+1 -11

@@ -35,7 +35,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

35 from vllm.model_executor.layers.logits_processor import LogitsProcessor

36 from vllm.model_executor.layers.quantization import QuantizationConfig

37 from vllm.model_executor.layers.rotary_embedding import get_rope

38 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

38 from vllm.model_executor.layers.vocab_parallel_embedding import (

39 ParallelLMHead, VocabParallelEmbedding)

40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -1394,7 +1393,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1393

1394 self.logits_processor = LogitsProcessor(config.embedding_size

1395 or config.vocab_size)

1397 - self.sampler = get_sampler()

1396

1397 self.make_empty_intermediate_tensors = (

1398 self.model.make_empty_intermediate_tensors)

@@ -1506,7 +1504,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1504 intermediate_tensors: Optional[IntermediateTensors] = None,

1505 inputs_embeds: Optional[torch.Tensor] = None,

1506 **kwargs: object,

1509 - ) -> SamplerOutput:

1507 + ) -> torch.Tensor:

1508

1509 if intermediate_tensors is not None:

1510 inputs_embeds = None

@@ -1532,14 +1530,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1530 sampling_metadata)

1531 return logits

1532

1535 - def sample(

1536 - self,

1537 - logits: torch.Tensor,

1538 - sampling_metadata: SamplingMetadata,

1539 - ) -> Optional[SamplerOutput]:

1540 - next_tokens = self.sampler(logits, sampling_metadata)

1541 - return next_tokens

1542 -

1533 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

1534

1535 loader = AutoWeightsLoader(self)

vllm/model_executor/models/mpt.py

-10

@@ -18,7 +18,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

18 RowParallelLinear)

19 from vllm.model_executor.layers.logits_processor import LogitsProcessor

20 from vllm.model_executor.layers.quantization import QuantizationConfig

21 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

21 from vllm.model_executor.layers.vocab_parallel_embedding import (

22 VocabParallelEmbedding)

23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -298,7 +297,6 @@ class MPTForCausalLM(nn.Module, SupportsPP):

297 prefix=maybe_prefix(prefix, "transformer"))

298 self.lm_head = self.transformer.wte

299 self.logits_processor = LogitsProcessor(config.vocab_size)

301 - self.sampler = get_sampler()

300 self.make_empty_intermediate_tensors = (

301 self.transformer.make_empty_intermediate_tensors)

302

@@ -325,14 +323,6 @@ class MPTForCausalLM(nn.Module, SupportsPP):

323 sampling_metadata)

324 return logits

325

328 - def sample(

329 - self,

330 - logits: torch.Tensor,

331 - sampling_metadata: SamplingMetadata,

332 - ) -> Optional[SamplerOutput]:

333 - next_tokens = self.sampler(logits, sampling_metadata)

334 - return next_tokens

335 -

326 def load_weights(self, weights: Iterable[Tuple[str,

327 torch.Tensor]]) -> Set[str]:

328 loader = AutoWeightsLoader(self)

vllm/model_executor/models/nemotron.py

-11

@@ -38,7 +38,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

38 from vllm.model_executor.layers.logits_processor import LogitsProcessor

39 from vllm.model_executor.layers.quantization import QuantizationConfig

40 from vllm.model_executor.layers.rotary_embedding import get_rope

41 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

41 from vllm.model_executor.layers.vocab_parallel_embedding import (

42 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

43 from vllm.model_executor.model_loader.weight_utils import (

@@ -416,8 +415,6 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

415 else:

416 self.lm_head = PPMissingLayer()

417

419 - self.sampler = get_sampler()

420 -

418 self.make_empty_intermediate_tensors = (

419 self.model.make_empty_intermediate_tensors)

420

@@ -444,14 +441,6 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

441 sampling_metadata)

442 return logits

443

447 - def sample(

448 - self,

449 - logits: torch.Tensor,

450 - sampling_metadata: SamplingMetadata,

451 - ) -> Optional[SamplerOutput]:

452 - next_tokens = self.sampler(logits, sampling_metadata)

453 - return next_tokens

454 -

444 def load_weights(self, weights: Iterable[Tuple[str,

445 torch.Tensor]]) -> Set[str]:

446 stacked_params_mapping = [

vllm/model_executor/models/nemotron_nas.py

-8

@@ -34,7 +34,6 @@ from vllm.distributed import get_pp_group

34 from vllm.model_executor.layers.layernorm import RMSNorm

35 from vllm.model_executor.layers.logits_processor import LogitsProcessor

36 from vllm.model_executor.layers.quantization import QuantizationConfig

37 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

37 from vllm.model_executor.layers.vocab_parallel_embedding import (

38 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

39 from vllm.model_executor.model_loader.weight_utils import (

@@ -408,8 +407,6 @@ class DeciLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, HasNoOps):

407 else:

408 self.lm_head = PPMissingLayer()

409

411 - self.sampler = get_sampler()

412 -

410 self.make_empty_intermediate_tensors = (

411 self.model.make_empty_intermediate_tensors)

412

@@ -439,11 +436,6 @@ class DeciLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, HasNoOps):

436 sampling_metadata)

437 return logits

438

442 - def sample(self, logits: torch.Tensor,

443 - sampling_metadata: SamplingMetadata) -> Optional[SamplerOutput]:

444 - next_tokens = self.sampler(logits, sampling_metadata)

445 - return next_tokens

446 -

439 def load_weights(self, weights: Iterable[Tuple[str,

440 torch.Tensor]]) -> Set[str]:

441 loader = AutoWeightsLoader(

vllm/model_executor/models/olmo.py

-10

@@ -39,7 +39,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

39 from vllm.model_executor.layers.logits_processor import LogitsProcessor

40 from vllm.model_executor.layers.quantization import QuantizationConfig

41 from vllm.model_executor.layers.rotary_embedding import get_rope

42 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

42 from vllm.model_executor.layers.vocab_parallel_embedding import (

43 ParallelLMHead, VocabParallelEmbedding)

44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -309,7 +308,6 @@ class OlmoForCausalLM(nn.Module, SupportsPP):

308 quant_config=quant_config,

309 )

310 self.logits_processor = LogitsProcessor(config.vocab_size)

312 - self.sampler = get_sampler()

311 self.make_empty_intermediate_tensors = (

312 self.model.make_empty_intermediate_tensors)

313

@@ -340,14 +338,6 @@ class OlmoForCausalLM(nn.Module, SupportsPP):

338 sampling_metadata)

339 return logits

340

343 - def sample(

344 - self,

345 - logits: torch.Tensor,

346 - sampling_metadata: SamplingMetadata,

347 - ) -> Optional[SamplerOutput]:

348 - next_tokens = self.sampler(logits, sampling_metadata)

349 - return next_tokens

350 -

341 def load_weights(self, weights: Iterable[Tuple[str,

342 torch.Tensor]]) -> Set[str]:

343 stacked_params_mapping = [

vllm/model_executor/models/olmo2.py

-10

@@ -42,7 +42,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

42 RowParallelLinear)

43 from vllm.model_executor.layers.logits_processor import LogitsProcessor

44 from vllm.model_executor.layers.rotary_embedding import get_rope

45 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

45 from vllm.model_executor.layers.vocab_parallel_embedding import (

46 ParallelLMHead, VocabParallelEmbedding)

47 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -339,7 +338,6 @@ class Olmo2ForCausalLM(nn.Module, SupportsPP):

338 prefix=maybe_prefix(prefix, "lm_head"),

339 )

340 self.logits_processor = LogitsProcessor(config.vocab_size)

342 - self.sampler = get_sampler()

341 self.make_empty_intermediate_tensors = (

342 self.model.make_empty_intermediate_tensors)

343

@@ -367,14 +365,6 @@ class Olmo2ForCausalLM(nn.Module, SupportsPP):

365 sampling_metadata)

366 return logits

367

370 - def sample(

371 - self,

372 - logits: torch.Tensor,

373 - sampling_metadata: SamplingMetadata,

374 - ) -> Optional[SamplerOutput]:

375 - next_tokens = self.sampler(logits, sampling_metadata)

376 - return next_tokens

377 -

368 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

369 stacked_params_mapping = [

370 # (param_name, shard_name, shard_id)

vllm/model_executor/models/olmoe.py

-10

@@ -31,7 +31,6 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

31 from vllm.model_executor.layers.logits_processor import LogitsProcessor

32 from vllm.model_executor.layers.quantization import QuantizationConfig

33 from vllm.model_executor.layers.rotary_embedding import get_rope

34 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

34 from vllm.model_executor.layers.vocab_parallel_embedding import (

35 ParallelLMHead, VocabParallelEmbedding)

36 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -417,7 +416,6 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):

416 config.hidden_size,

417 quant_config=quant_config)

418 self.logits_processor = LogitsProcessor(config.vocab_size)

420 - self.sampler = get_sampler()

419

420 self.make_empty_intermediate_tensors = (

421 self.model.make_empty_intermediate_tensors)

@@ -442,14 +440,6 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):

440 sampling_metadata)

441 return logits

442

445 - def sample(

446 - self,

447 - logits: Optional[torch.Tensor],

448 - sampling_metadata: SamplingMetadata,

449 - ) -> Optional[SamplerOutput]:

450 - next_tokens = self.sampler(logits, sampling_metadata)

451 - return next_tokens

452 -

443 def load_weights(self, weights: Iterable[Tuple[str,

444 torch.Tensor]]) -> Set[str]:

445 loader = AutoWeightsLoader(

vllm/model_executor/models/opt.py

-10

@@ -35,7 +35,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

35 RowParallelLinear)

36 from vllm.model_executor.layers.logits_processor import LogitsProcessor

37 from vllm.model_executor.layers.quantization import QuantizationConfig

38 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

38 from vllm.model_executor.layers.vocab_parallel_embedding import (

39 ParallelLMHead, VocabParallelEmbedding)

40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -375,7 +374,6 @@ class OPTForCausalLM(nn.Module, SupportsPP):

374 self.lm_head = ParallelLMHead(config.vocab_size,

375 config.word_embed_proj_dim)

376 self.logits_processor = LogitsProcessor(config.vocab_size)

378 - self.sampler = get_sampler()

377 self.make_empty_intermediate_tensors = (

378 self.model.make_empty_intermediate_tensors)

379

@@ -402,14 +400,6 @@ class OPTForCausalLM(nn.Module, SupportsPP):

400 sampling_metadata)

401 return logits

402

405 - def sample(

406 - self,

407 - logits: torch.Tensor,

408 - sampling_metadata: SamplingMetadata,

409 - ) -> Optional[SamplerOutput]:

410 - next_tokens = self.sampler(logits, sampling_metadata)

411 - return next_tokens

412 -

403 def load_weights(self, weights: Iterable[Tuple[str,

404 torch.Tensor]]) -> Set[str]:

405 loader = AutoWeightsLoader(

vllm/model_executor/models/orion.py

-10

@@ -22,7 +22,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

22 from vllm.model_executor.layers.logits_processor import LogitsProcessor

23 from vllm.model_executor.layers.quantization import QuantizationConfig

24 from vllm.model_executor.layers.rotary_embedding import get_rope

25 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

25 from vllm.model_executor.layers.vocab_parallel_embedding import (

26 ParallelLMHead, VocabParallelEmbedding)

27 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -316,7 +315,6 @@ class OrionForCausalLM(nn.Module, SupportsPP):

315 if self.config.tie_word_embeddings:

316 self.lm_head.weight = self.model.embed_tokens.weight

317 self.logits_processor = LogitsProcessor(config.vocab_size)

319 - self.sampler = get_sampler()

318 self.make_empty_intermediate_tensors = (

319 self.model.make_empty_intermediate_tensors)

320

@@ -343,14 +341,6 @@ class OrionForCausalLM(nn.Module, SupportsPP):

341 sampling_metadata)

342 return logits

343

346 - def sample(

347 - self,

348 - logits: torch.Tensor,

349 - sampling_metadata: SamplingMetadata,

350 - ) -> Optional[SamplerOutput]:

351 - next_tokens = self.sampler(logits, sampling_metadata)

352 - return next_tokens

353 -

344 def load_weights(self, weights: Iterable[Tuple[str,

345 torch.Tensor]]) -> Set[str]:

346 loader = AutoWeightsLoader(

vllm/model_executor/models/paligemma.py

+1 -13

@@ -8,7 +8,6 @@ from transformers import BatchFeature, PaliGemmaConfig

8

9 from vllm.config import VllmConfig

10 from vllm.logger import init_logger

11 -from vllm.model_executor.layers.sampler import SamplerOutput

11 from vllm.model_executor.sampling_metadata import SamplingMetadata

12 from vllm.multimodal import MULTIMODAL_REGISTRY

13 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -260,10 +259,6 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

259 self.make_empty_intermediate_tensors = (

260 self.language_model.make_empty_intermediate_tensors)

261

263 - @property

264 - def sampler(self):

265 - return self.language_model.sampler

266 -

262 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:

263 h = w = self.config.vision_config.image_size

264 expected_dims = (3, h, w)

@@ -369,7 +364,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

364 positions: torch.Tensor,

365 intermediate_tensors: Optional[IntermediateTensors] = None,

366 inputs_embeds: Optional[torch.Tensor] = None,

372 - **kwargs: object) -> Union[SamplerOutput, IntermediateTensors]:

367 + **kwargs: object) -> IntermediateTensors:

368 if intermediate_tensors is not None:

369 inputs_embeds = None

370

@@ -396,13 +391,6 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

391 return self.language_model.compute_logits(hidden_states,

392 sampling_metadata)

393

399 - def sample(

400 - self,

401 - logits: torch.Tensor,

402 - sampling_metadata: SamplingMetadata,

403 - ) -> Optional[SamplerOutput]:

404 - return self.language_model.sample(logits, sampling_metadata)

405 -

394 def load_weights(self, weights: Iterable[Tuple[str,

395 torch.Tensor]]) -> Set[str]:

396 loader = AutoWeightsLoader(self)

vllm/model_executor/models/persimmon.py

-10

@@ -38,7 +38,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

38 from vllm.model_executor.layers.logits_processor import LogitsProcessor

39 from vllm.model_executor.layers.quantization import QuantizationConfig

40 from vllm.model_executor.layers.rotary_embedding import get_rope

41 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

41 from vllm.model_executor.layers.vocab_parallel_embedding import (

42 ParallelLMHead, VocabParallelEmbedding)

43 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -306,7 +305,6 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):

305 config.hidden_size,

306 bias=False)

307 self.logits_processor = LogitsProcessor(config.vocab_size)

309 - self.sampler = get_sampler()

308 self.make_empty_intermediate_tensors = (

309 self.model.make_empty_intermediate_tensors)

310

@@ -337,14 +335,6 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):

335 sampling_metadata)

336 return logits

337

340 - def sample(

341 - self,

342 - logits: torch.Tensor,

343 - sampling_metadata: SamplingMetadata,

344 - ) -> Optional[SamplerOutput]:

345 - next_tokens = self.sampler(logits, sampling_metadata)

346 - return next_tokens

347 -

338 def load_weights(self, weights: Iterable[Tuple[str,

339 torch.Tensor]]) -> Set[str]:

340 loader = AutoWeightsLoader(self)

vllm/model_executor/models/phi.py

-10

@@ -53,7 +53,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

53 from vllm.model_executor.layers.logits_processor import LogitsProcessor

54 from vllm.model_executor.layers.quantization import QuantizationConfig

55 from vllm.model_executor.layers.rotary_embedding import get_rope

56 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

56 from vllm.model_executor.layers.vocab_parallel_embedding import (

57 ParallelLMHead, VocabParallelEmbedding)

58 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -322,7 +321,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

321 bias=True,

322 quant_config=quant_config)

323 self.logits_processor = LogitsProcessor(config.vocab_size)

325 - self.sampler = get_sampler()

324 self.make_empty_intermediate_tensors = (

325 self.model.make_empty_intermediate_tensors)

326

@@ -350,14 +348,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

348 sampling_metadata, self.lm_head.bias)

349 return logits

350

353 - def sample(

354 - self,

355 - logits: torch.Tensor,

356 - sampling_metadata: SamplingMetadata,

357 - ) -> Optional[SamplerOutput]:

358 - next_tokens = self.sampler(logits, sampling_metadata)

359 - return next_tokens

360 -

351 def load_weights(self, weights: Iterable[Tuple[str,

352 torch.Tensor]]) -> Set[str]:

353 loader = AutoWeightsLoader(self)

vllm/model_executor/models/phi3_small.py

+1 -12

@@ -17,7 +17,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

17 from vllm.model_executor.layers.logits_processor import LogitsProcessor

18 from vllm.model_executor.layers.quantization import QuantizationConfig

19 from vllm.model_executor.layers.rotary_embedding import get_rope

20 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

20 from vllm.model_executor.layers.vocab_parallel_embedding import (

21 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -396,7 +395,6 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):

395 if self.config.tie_word_embeddings:

396 self.lm_head.weight = self.model.embed_tokens.weight

397 self.logits_processor = LogitsProcessor(config.vocab_size)

399 - self.sampler = get_sampler()

398 self.make_empty_intermediate_tensors = (

399 self.model.make_empty_intermediate_tensors)

400

@@ -437,6 +435,7 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):

435 sampling_metadata)

436 if self.dummy_token_indices is not None and logits is not None:

437 logits.index_fill_(-1, self.dummy_token_indices, -torch.inf)

438 + logits = logits / self.mup_width_multiplier

439 return logits

440

441 def forward(

@@ -455,16 +454,6 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):

454 output_hidden_states = output_hidden_states

455 return output_hidden_states

456

458 - def sample(

459 - self,

460 - logits: torch.Tensor,

461 - sampling_metadata: SamplingMetadata,

462 - ) -> Optional[SamplerOutput]:

463 -

464 - next_tokens = self.sampler(logits / self.mup_width_multiplier,

465 - sampling_metadata)

466 - return next_tokens

467 -

457 def load_weights(self, weights: Iterable[Tuple[str,

458 torch.Tensor]]) -> Set[str]:

459 loader = AutoWeightsLoader(

vllm/model_executor/models/phi3v.py

-16

@@ -16,7 +16,6 @@

16 # limitations under the License.

17 import re

18 from collections.abc import Iterable, Mapping, Sequence

19 -from functools import cached_property

19 from typing import Any, List, Literal, Optional, Set, Tuple, TypedDict, Union

20

21 import torch

@@ -27,7 +26,6 @@ from transformers import (BatchFeature, CLIPVisionConfig, PretrainedConfig,

26 from vllm.config import VllmConfig

27 from vllm.logger import init_logger

28 from vllm.model_executor.layers.quantization import QuantizationConfig

30 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

29 from vllm.model_executor.layers.vocab_parallel_embedding import (

30 VocabParallelEmbedding)

31 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -555,13 +553,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

553 self.make_empty_intermediate_tensors = (

554 self.language_model.make_empty_intermediate_tensors)

555

558 - @cached_property

559 - def sampler(self):

560 - if hasattr(self.language_model, "sampler"):

561 - return self.language_model.sampler

562 -

563 - return get_sampler()

564 -

556 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:

557 expected_dims = (2, )

558

@@ -716,13 +707,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

707 return self.language_model.compute_logits(hidden_states,

708 sampling_metadata)

709

719 - def sample(

720 - self,

721 - logits: torch.Tensor,

722 - sampling_metadata: SamplingMetadata,

723 - ) -> Optional[SamplerOutput]:

724 - return self.language_model.sample(logits, sampling_metadata)

725 -

710 def load_weights(self, weights: Iterable[Tuple[str,

711 torch.Tensor]]) -> Set[str]:

712

vllm/model_executor/models/phi4mm.py

-10

@@ -13,7 +13,6 @@ from vllm.config import VllmConfig

13 from vllm.distributed import get_pp_group

14 from vllm.model_executor.layers.logits_processor import LogitsProcessor

15 from vllm.model_executor.layers.quantization import QuantizationConfig

16 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

16 from vllm.model_executor.layers.vocab_parallel_embedding import (

17 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)

18 from vllm.model_executor.models.llama import LlamaModel

@@ -968,7 +967,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

967 logit_scale = getattr(config, "logit_scale", 1.0)

968 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

969 config.vocab_size, logit_scale)

971 - self.sampler = get_sampler()

970

971 def _parse_and_validate_audio_input(

972 self, **kwargs: object) -> Optional[Phi4MMAudioInputs]:

@@ -1244,14 +1242,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1242 sampling_metadata)

1243 return logits

1244

1247 - def sample(

1248 - self,

1249 - logits: torch.Tensor,

1250 - sampling_metadata: SamplingMetadata,

1251 - ) -> Optional[SamplerOutput]:

1252 - next_tokens = self.sampler(logits, sampling_metadata)

1253 - return next_tokens

1254 -

1245 def load_weights(self, weights: Iterable[Tuple[str,

1246 torch.Tensor]]) -> None:

1247 weights = ((name, data) for name, data in weights

vllm/model_executor/models/phimoe.py

-10

@@ -40,7 +40,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

40 from vllm.model_executor.layers.quantization.base_config import (

41 QuantizationConfig)

42 from vllm.model_executor.layers.rotary_embedding import get_rope

43 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

43 from vllm.model_executor.layers.vocab_parallel_embedding import (

44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

45 from vllm.model_executor.model_loader.weight_utils import (

@@ -634,7 +633,6 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

633 )

634 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

635 config.vocab_size)

637 - self.sampler = get_sampler()

636

637 self.make_empty_intermediate_tensors = (

638 self.model.make_empty_intermediate_tensors)

@@ -659,14 +657,6 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

657 sampling_metadata)

658 return logits

659

662 - def sample(

663 - self,

664 - logits: Optional[torch.Tensor],

665 - sampling_metadata: SamplingMetadata,

666 - ) -> Optional[SamplerOutput]:

667 - next_tokens = self.sampler(logits, sampling_metadata)

668 - return next_tokens

669 -

660 def load_weights(self, weights: Iterable[Tuple[str,

661 torch.Tensor]]) -> Set[str]:

662 loader = AutoWeightsLoader(

vllm/model_executor/models/pixtral.py

-15

@@ -28,7 +28,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

28 QKVParallelLinear,

29 RowParallelLinear)

30 from vllm.model_executor.layers.quantization import QuantizationConfig

31 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

31 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

32 from vllm.model_executor.sampling_metadata import SamplingMetadata

33 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

@@ -331,13 +330,6 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

330 self.make_empty_intermediate_tensors = (

331 self.language_model.make_empty_intermediate_tensors)

332

334 - @cached_property

335 - def sampler(self):

336 - if hasattr(self.language_model, "sampler"):

337 - return self.language_model.sampler

338 -

339 - return get_sampler()

340 -

333 def _parse_and_validate_image_input(

334 self, **kwargs: object) -> Optional[PixtralImagePixelInputs]:

335 images = kwargs.pop("images", None)

@@ -441,13 +433,6 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

433 return self.language_model.compute_logits(hidden_states,

434 sampling_metadata)

435

444 - def sample(

445 - self,

446 - logits: torch.Tensor,

447 - sampling_metadata: SamplingMetadata,

448 - ) -> Optional[SamplerOutput]:

449 - return self.language_model.sample(logits, sampling_metadata)

450 -

436 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

437

438 def is_vision_encoder_weights(weight: Tuple[str, torch.Tensor]):

vllm/model_executor/models/plamo2.py

-10

@@ -24,7 +24,6 @@ from vllm.model_executor.layers.mamba.ops.mamba_ssm import (

24 selective_scan_fn, selective_state_update)

25 from vllm.model_executor.layers.quantization import QuantizationConfig

26 from vllm.model_executor.layers.rotary_embedding import get_rope

27 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

27 from vllm.model_executor.layers.vocab_parallel_embedding import (

28 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

29 from vllm.model_executor.model_loader.weight_utils import (

@@ -628,7 +627,6 @@ class Plamo2ForCausalLM(Plamo2PreTrainedModel, HasInnerState, IsHybrid,

627

628 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,

629 self.config.vocab_size)

631 - self.sampler = get_sampler()

630

631 # Initialize weights and apply final processing

632 self.post_init()

@@ -684,14 +682,6 @@ class Plamo2ForCausalLM(Plamo2PreTrainedModel, HasInnerState, IsHybrid,

682 sampling_metadata)

683 return logits

684

687 - def sample(

688 - self,

689 - logits: Optional[torch.Tensor],

690 - sampling_metadata: SamplingMetadata,

691 - ) -> Optional[SamplerOutput]:

692 - next_tokens = self.sampler(logits, sampling_metadata)

693 - return next_tokens

694 -

685 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):

686 params_dict = dict(self.named_parameters())

687 for name, loaded_weight in weights:

vllm/model_executor/models/qwen.py

-10

@@ -24,7 +24,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

24 from vllm.model_executor.layers.logits_processor import LogitsProcessor

25 from vllm.model_executor.layers.quantization import QuantizationConfig

26 from vllm.model_executor.layers.rotary_embedding import get_rope

27 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

27 from vllm.model_executor.layers.vocab_parallel_embedding import (

28 ParallelLMHead, VocabParallelEmbedding)

29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -273,7 +272,6 @@ class QWenBaseModel(nn.Module):

272 if self.config.tie_word_embeddings:

273 self.lm_head.weight = self.transformer.wte.weight

274 self.logits_processor = LogitsProcessor(config.vocab_size)

276 - self.sampler = get_sampler()

275 self.make_empty_intermediate_tensors = (

276 self.transformer.make_empty_intermediate_tensors)

277

@@ -286,14 +284,6 @@ class QWenBaseModel(nn.Module):

284 sampling_metadata)

285 return logits

286

289 - def sample(

290 - self,

291 - logits: torch.Tensor,

292 - sampling_metadata: SamplingMetadata,

293 - ) -> Optional[SamplerOutput]:

294 - next_tokens = self.sampler(logits, sampling_metadata)

295 - return next_tokens

296 -

287 def load_weights(self, weights: Iterable[Tuple[str,

288 torch.Tensor]]) -> Set[str]:

289 stacked_params_mapping = [

vllm/model_executor/models/qwen2.py

-10

@@ -43,7 +43,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor

43 from vllm.model_executor.layers.pooler import Pooler, PoolingType

44 from vllm.model_executor.layers.quantization import QuantizationConfig

45 from vllm.model_executor.layers.rotary_embedding import get_rope

46 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

46 from vllm.model_executor.layers.vocab_parallel_embedding import (

47 ParallelLMHead, VocabParallelEmbedding)

48 from vllm.model_executor.model_loader.weight_utils import (

@@ -450,7 +449,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

449 self.lm_head = PPMissingLayer()

450

451 self.logits_processor = LogitsProcessor(config.vocab_size)

453 - self.sampler = get_sampler()

452

453 self.make_empty_intermediate_tensors = (

454 self.model.make_empty_intermediate_tensors)

@@ -478,14 +476,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

476 sampling_metadata)

477 return logits

478

481 - def sample(

482 - self,

483 - logits: torch.Tensor,

484 - sampling_metadata: SamplingMetadata,

485 - ) -> Optional[SamplerOutput]:

486 - next_tokens = self.sampler(logits, sampling_metadata)

487 - return next_tokens

488 -

479 def load_weights(self, weights: Iterable[Tuple[str,

480 torch.Tensor]]) -> Set[str]:

481 loader = AutoWeightsLoader(

vllm/model_executor/models/qwen2_5_omni_thinker.py

+1 -16

@@ -22,7 +22,7 @@

22 """Inference-only Qwen2.5-Omni model (thinker part)."""

23

24 from copy import copy

25 -from functools import cached_property, partial

25 +from functools import partial

26 from typing import (Any, Dict, Iterable, List, Mapping, Optional, Sequence,

27 Set, Tuple, Union)

28

@@ -40,7 +40,6 @@ from transformers.models.whisper import WhisperFeatureExtractor

40 from vllm.config import VllmConfig

41 from vllm.logger import init_logger

42 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding

43 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

43 from vllm.model_executor.models.qwen2_5_vl import (

44 Qwen2_5_VisionTransformer, Qwen2_5_VLImageEmbeddingInputs,

45 Qwen2_5_VLImageInputs, Qwen2_5_VLImagePixelInputs,

@@ -790,13 +789,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

789 self.make_empty_intermediate_tensors = (

790 self.language_model.make_empty_intermediate_tensors)

791

793 - @cached_property

794 - def sampler(self):

795 - if hasattr(self.language_model, "sampler"):

796 - return self.language_model.sampler

797 -

798 - return get_sampler()

799 -

792 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

793 mm_input_by_modality = {}

794

@@ -937,13 +929,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

929 return self.language_model.compute_logits(hidden_states,

930 sampling_metadata)

931

940 - def sample(

941 - self,

942 - logits: torch.Tensor,

943 - sampling_metadata: SamplingMetadata,

944 - ) -> Optional[SamplerOutput]:

945 - return self.language_model.sample(logits, sampling_metadata)

946 -

932 def load_weights(self, weights: Iterable[Tuple[str,

933 torch.Tensor]]) -> Set[str]:

934 loader = AutoWeightsLoader(

vllm/model_executor/models/qwen2_5_vl.py

+1 -16

@@ -24,7 +24,7 @@

24 # See the License for the specific language governing permissions and

25 # limitations under the License.

26 """Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""

27 -from functools import cached_property, partial

27 +from functools import partial

28 from typing import (Callable, Iterable, List, Literal, Mapping, Optional, Set,

29 Tuple, TypedDict, Union)

30

@@ -51,7 +51,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

51 from vllm.model_executor.layers.quantization.gptq import GPTQConfig

52 from vllm.model_executor.layers.quantization.gptq_marlin import (

53 GPTQMarlinConfig)

54 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

54 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

55 from vllm.model_executor.models.module_mapping import MultiModelKeys

56 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -833,13 +832,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

832 self.make_empty_intermediate_tensors = (

833 self.language_model.make_empty_intermediate_tensors)

834

836 - @cached_property

837 - def sampler(self):

838 - if hasattr(self.language_model, "sampler"):

839 - return self.language_model.sampler

840 -

841 - return get_sampler()

842 -

835 def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

836 # GPTQ configs do not have a list of ignored modules, however AutoGPTQ

837 # seems to avoid vision encoder sections for some models.

@@ -1127,13 +1119,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1119 return self.language_model.compute_logits(hidden_states,

1120 sampling_metadata)

1121

1130 - def sample(

1131 - self,

1132 - logits: torch.Tensor,

1133 - sampling_metadata: SamplingMetadata,

1134 - ) -> Optional[SamplerOutput]:

1135 - return self.language_model.sample(logits, sampling_metadata)

1136 -

1122 def load_weights(self, weights: Iterable[Tuple[str,

1123 torch.Tensor]]) -> Set[str]:

1124

vllm/model_executor/models/qwen2_audio.py

-16

@@ -22,7 +22,6 @@

22 # limitations under the License.

23 """Inference-only Qwen2-Audio model compatible with HuggingFace weights."""

24 from collections.abc import Iterable, Mapping, Sequence

25 -from functools import cached_property

25 from typing import Any, Optional, Set, Tuple, TypedDict, Union

26

27 import torch

@@ -34,7 +33,6 @@ from transformers.models.qwen2_audio import (Qwen2AudioConfig,

33 from transformers.models.whisper import WhisperFeatureExtractor

34

35 from vllm.config import VllmConfig

37 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

36 from vllm.model_executor.sampling_metadata import SamplingMetadata

37 from vllm.multimodal import MULTIMODAL_REGISTRY

38 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -267,13 +265,6 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

265 self.make_empty_intermediate_tensors = (

266 self.language_model.make_empty_intermediate_tensors)

267

270 - @cached_property

271 - def sampler(self):

272 - if hasattr(self.language_model, "sampler"):

273 - return self.language_model.sampler

274 -

275 - return get_sampler()

276 -

268 def _validate_and_reshape_mm_tensor(self, mm_input: object,

269 name: str) -> torch.Tensor:

270 if not isinstance(mm_input, (torch.Tensor, list)):

@@ -405,13 +396,6 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

396 return self.language_model.compute_logits(hidden_states,

397 sampling_metadata)

398

408 - def sample(

409 - self,

410 - logits: torch.Tensor,

411 - sampling_metadata: SamplingMetadata,

412 - ) -> Optional[SamplerOutput]:

413 - return self.language_model.sample(logits, sampling_metadata)

414 -

399 def load_weights(self, weights: Iterable[Tuple[str,

400 torch.Tensor]]) -> Set[str]:

401 loader = AutoWeightsLoader(self)

vllm/model_executor/models/qwen2_moe.py

-10

@@ -47,7 +47,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

47 from vllm.model_executor.layers.logits_processor import LogitsProcessor

48 from vllm.model_executor.layers.quantization import QuantizationConfig

49 from vllm.model_executor.layers.rotary_embedding import get_rope

50 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

50 from vllm.model_executor.layers.vocab_parallel_embedding import (

51 ParallelLMHead, VocabParallelEmbedding)

52 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -497,7 +496,6 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):

496 if self.config.tie_word_embeddings:

497 self.lm_head.weight = self.model.embed_tokens.weight

498 self.logits_processor = LogitsProcessor(config.vocab_size)

500 - self.sampler = get_sampler()

499 self.make_empty_intermediate_tensors = (

500 self.model.make_empty_intermediate_tensors)

501

@@ -524,14 +522,6 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):

522 sampling_metadata)

523 return logits

524

527 - def sample(

528 - self,

529 - logits: Optional[torch.Tensor],

530 - sampling_metadata: SamplingMetadata,

531 - ) -> Optional[SamplerOutput]:

532 - next_tokens = self.sampler(logits, sampling_metadata)

533 - return next_tokens

534 -

525 def load_weights(self, weights: Iterable[Tuple[str,

526 torch.Tensor]]) -> Set[str]:

527 loader = AutoWeightsLoader(

vllm/model_executor/models/qwen2_vl.py

+1 -16

@@ -24,7 +24,7 @@

24 # limitations under the License.

25 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""

26 from collections.abc import Iterable, Mapping, Sequence

27 -from functools import cached_property, partial

27 +from functools import partial

28 from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict,

29 Union)

30

@@ -51,7 +51,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

51 from vllm.model_executor.layers.quantization.gptq import GPTQConfig

52 from vllm.model_executor.layers.quantization.gptq_marlin import (

53 GPTQMarlinConfig)

54 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

54 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

55 from vllm.model_executor.models.module_mapping import MultiModelKeys

56 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -1112,13 +1111,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1111 self.make_empty_intermediate_tensors = (

1112 self.language_model.make_empty_intermediate_tensors)

1113

1115 - @cached_property

1116 - def sampler(self):

1117 - if hasattr(self.language_model, "sampler"):

1118 - return self.language_model.sampler

1119 -

1120 - return get_sampler()

1121 -

1114 def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):

1115 # GPTQ configs do not have a list of ignored modules, however AutoGPTQ

1116 # seems to avoid vision encoder sections for some models.

@@ -1400,13 +1392,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1392 return self.language_model.compute_logits(hidden_states,

1393 sampling_metadata)

1394

1403 - def sample(

1404 - self,

1405 - logits: torch.Tensor,

1406 - sampling_metadata: SamplingMetadata,

1407 - ) -> Optional[SamplerOutput]:

1408 - return self.language_model.sample(logits, sampling_metadata)

1409 -

1395 def load_weights(self, weights: Iterable[Tuple[str,

1396 torch.Tensor]]) -> Set[str]:

1397

vllm/model_executor/models/qwen3.py

-10

@@ -38,7 +38,6 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,

38 from vllm.model_executor.layers.logits_processor import LogitsProcessor

39 from vllm.model_executor.layers.quantization import QuantizationConfig

40 from vllm.model_executor.layers.rotary_embedding import get_rope

41 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

41 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

42 from vllm.model_executor.sampling_metadata import SamplingMetadata

43 from vllm.sequence import IntermediateTensors

@@ -283,7 +282,6 @@ class Qwen3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

282 self.lm_head = PPMissingLayer()

283

284 self.logits_processor = LogitsProcessor(config.vocab_size)

286 - self.sampler = get_sampler()

285

286 self.make_empty_intermediate_tensors = (

287 self.model.make_empty_intermediate_tensors)

@@ -311,14 +309,6 @@ class Qwen3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

309 sampling_metadata)

310 return logits

311

314 - def sample(

315 - self,

316 - logits: torch.Tensor,

317 - sampling_metadata: SamplingMetadata,

318 - ) -> Optional[SamplerOutput]:

319 - next_tokens = self.sampler(logits, sampling_metadata)

320 - return next_tokens

321 -

312 def load_weights(self, weights: Iterable[Tuple[str,

313 torch.Tensor]]) -> Set[str]:

314 loader = AutoWeightsLoader(

vllm/model_executor/models/qwen3_moe.py

-10

@@ -44,7 +44,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

44 from vllm.model_executor.layers.logits_processor import LogitsProcessor

45 from vllm.model_executor.layers.quantization import QuantizationConfig

46 from vllm.model_executor.layers.rotary_embedding import get_rope

47 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

47 from vllm.model_executor.layers.vocab_parallel_embedding import (

48 ParallelLMHead, VocabParallelEmbedding)

49 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -494,7 +493,6 @@ class Qwen3MoeForCausalLM(nn.Module, SupportsPP):

493 if self.config.tie_word_embeddings:

494 self.lm_head.weight = self.model.embed_tokens.weight

495 self.logits_processor = LogitsProcessor(config.vocab_size)

497 - self.sampler = get_sampler()

496 self.make_empty_intermediate_tensors = (

497 self.model.make_empty_intermediate_tensors)

498

@@ -521,14 +519,6 @@ class Qwen3MoeForCausalLM(nn.Module, SupportsPP):

519 sampling_metadata)

520 return logits

521

524 - def sample(

525 - self,

526 - logits: Optional[torch.Tensor],

527 - sampling_metadata: SamplingMetadata,

528 - ) -> Optional[SamplerOutput]:

529 - next_tokens = self.sampler(logits, sampling_metadata)

530 - return next_tokens

531 -

522 def load_weights(self, weights: Iterable[Tuple[str,

523 torch.Tensor]]) -> Set[str]:

524 loader = AutoWeightsLoader(

vllm/model_executor/models/skyworkr1v.py

+1 -17

@@ -8,7 +8,6 @@

8 # --------------------------------------------------------

9 from abc import ABC, abstractmethod

10 from collections.abc import Iterable, Mapping, Sequence

11 -from functools import cached_property

11 from typing import Literal, Optional, Set, Tuple, TypedDict, TypeVar, Union

12

13 import torch

@@ -21,7 +20,6 @@ from vllm.config import VllmConfig

20 from vllm.model_executor.layers.linear import ReplicatedLinear

21 from vllm.model_executor.layers.quantization import QuantizationConfig

22 from vllm.model_executor.layers.quantization.awq import AWQConfig

24 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

23 from vllm.model_executor.models.intern_vit import (InternVisionModel,

24 InternVisionPatchModel)

25 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -699,13 +697,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

697 (llm_quant_config is not None):

698 quant_config.modules_to_not_convert.append("vision_model")

699

702 - @cached_property

703 - def sampler(self):

704 - if hasattr(self.language_model, "sampler"):

705 - return self.language_model.sampler

706 -

707 - return get_sampler()

708 -

700 def _init_vision_model(

701 self,

702 config: PretrainedConfig,

@@ -908,7 +899,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

899 intermediate_tensors: Optional[IntermediateTensors] = None,

900 inputs_embeds: Optional[torch.Tensor] = None,

901 **kwargs: object,

911 - ) -> Union[SamplerOutput, IntermediateTensors]:

902 + ) -> IntermediateTensors:

903

904 if intermediate_tensors is not None:

905 input_ids = None

@@ -946,13 +937,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

937 return self.language_model.compute_logits(hidden_states,

938 sampling_metadata)

939

949 - def sample(

950 - self,

951 - logits: torch.Tensor,

952 - sampling_metadata: SamplingMetadata,

953 - ) -> Optional[SamplerOutput]:

954 - return self.language_model.sample(logits, sampling_metadata)

955 -

940 def load_weights(self, weights: Iterable[Tuple[str,

941 torch.Tensor]]) -> Set[str]:

942 skip_prefixes = [

vllm/model_executor/models/solar.py