@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d58268c56a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Make v1 more testable (#9888)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Nov 6, 2024 at 12:57 UTC d58268c56a8ee0eb01c30e7ab7c07c934e1791c2
75 files changed +243 -165
Dockerfile
+3
@@ -191,6 +191,9 @@ ADD . /vllm-workspace/
191 RUN --mount=type=cache,target=/root/.cache/pip \
192 python3 -m pip install -r requirements-dev.txt
193
194 +# Copy in the v1 package for testing (it isn't distributed yet)
195 +COPY vllm/v1 /usr/local/lib/python3.12/dist-packages/vllm/v1
196 +
197 # doc requires source code
198 # we hide them inside `test_docs/` , so that this source code
199 # will not be imported by other tests
pyproject.toml
+1
@@ -97,4 +97,5 @@ markers = [
97 "skip_global_cleanup",
98 "core_model: run this model test in each PR instead of just daily",
99 "distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
100 + "skip_v1: do not run this test with v1",
101 ]
tests/conftest.py
+18
@@ -5,6 +5,7 @@ from collections import UserList
5 from enum import Enum
6 from typing import (Any, Callable, Dict, List, Optional, Tuple, Type,
7 TypedDict, TypeVar, Union)
8 +from unittest.mock import patch
9
10 import numpy as np
11 import pytest
@@ -108,6 +109,23 @@ VIDEO_ASSETS = _VideoAssets()
109 """Singleton instance of :class:`_VideoAssets`."""
110
111
112 +@pytest.fixture(params=[True, False])
113 +def run_with_both_engines(request):
114 + # Automatically runs tests twice, once with V1 and once without
115 + use_v1 = request.param
116 + # Tests decorated with `@skip_v1` are only run without v1
117 + skip_v1 = request.node.get_closest_marker("skip_v1")
118 +
119 + if use_v1:
120 + if skip_v1:
121 + pytest.skip("Skipping test on vllm V1")
122 + with patch('vllm.envs.VLLM_USE_V1', True):
123 + yield
124 + else:
125 + with patch('vllm.envs.VLLM_USE_V1', False):
126 + yield
127 +
128 +
129 @pytest.fixture(autouse=True)
130 def init_test_http_connection():
131 # pytest_asyncio may use a different event loop per test
tests/entrypoints/llm/test_prompt_validation.py
+9
@@ -3,12 +3,21 @@ import pytest
3 from vllm import LLM
4
5
6 +@pytest.fixture(autouse=True)
7 +def v1(run_with_both_engines):
8 + # Simple autouse wrapper to run both engines for each test
9 + # This can be promoted up to conftest.py to run for every
10 + # test in a package
11 + pass
12 +
13 +
14 def test_empty_prompt():
15 llm = LLM(model="gpt2", enforce_eager=True)
16 with pytest.raises(ValueError, match='Prompt cannot be empty'):
17 llm.generate([""])
18
19
20 +@pytest.mark.skip_v1
21 def test_out_of_vocab_token():
22 llm = LLM(model="gpt2", enforce_eager=True)
23 with pytest.raises(ValueError, match='out of vocabulary'):
tests/kernels/test_attention_selector.py
+2
@@ -44,6 +44,8 @@ def test_env(name: str, device: str, monkeypatch):
44
45 def test_flash_attn(monkeypatch):
46 """Test FlashAttn validation."""
47 + # TODO: When testing for v1, pipe in `use_v1` as an argument to
48 + # which_attn_to_use
49
50 override_backend_env_variable(monkeypatch, STR_FLASH_ATTN_VAL)
51
tests/kernels/test_encoder_decoder_attn.py
+2 -2
@@ -16,7 +16,7 @@ from tests.kernels.utils import *
16 from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
17 AttentionType)
18 from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19 -from vllm.attention.selector import (_Backend, get_attn_backend,
19 +from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
20 global_force_attn_backend_context_manager)
21 from vllm.forward_context import set_forward_context
22 from vllm.platforms import current_platform
@@ -774,7 +774,7 @@ def set_reset_environment(attn_backend):
774 default_dtype = torch.get_default_dtype()
775 if attn_backend.name == 'FLASH_ATTN':
776 torch.set_default_dtype(torch.bfloat16)
777 - get_attn_backend.cache_clear()
777 + _cached_get_attn_backend.cache_clear()
778 yield
779 # Reset the torch datatype to what it was before the test
780 # so as not to impact the remaining tests.
vllm/attention/selector.py
+33 -10
@@ -89,7 +89,6 @@ def get_global_forced_attn_backend() -> Optional[_Backend]:
89 return forced_attn_backend
90
91
92 -@lru_cache(maxsize=None)
92 def get_attn_backend(
93 head_size: int,
94 dtype: torch.dtype,
@@ -99,6 +98,31 @@ def get_attn_backend(
98 is_blocksparse: bool = False,
99 ) -> Type[AttentionBackend]:
100 """Selects which attention backend to use and lazily imports it."""
101 + # Accessing envs.* behind an @lru_cache decorator can cause the wrong
102 + # value to be returned from the cache if the value changes between calls.
103 + # To avoid this, we read envs.VLLM_USE_V1 here and pass it explicitly to the
104 + # private function.
105 + return _cached_get_attn_backend(
106 + head_size=head_size,
107 + dtype=dtype,
108 + kv_cache_dtype=kv_cache_dtype,
109 + block_size=block_size,
110 + is_attention_free=is_attention_free,
111 + is_blocksparse=is_blocksparse,
112 + use_v1=envs.VLLM_USE_V1,
113 + )
114 +
115 +
116 +@lru_cache(maxsize=None)
117 +def _cached_get_attn_backend(
118 + head_size: int,
119 + dtype: torch.dtype,
120 + kv_cache_dtype: Optional[str],
121 + block_size: int,
122 + is_attention_free: bool,
123 + is_blocksparse: bool = False,
124 + use_v1: bool = False,
125 +) -> Type[AttentionBackend]:
126 if is_blocksparse:
127 logger.info("Using BlocksparseFlashAttention backend.")
128 from vllm.attention.backends.blocksparse_attn import (
@@ -106,7 +130,7 @@ def get_attn_backend(
130 return BlocksparseFlashAttentionBackend
131
132 backend = which_attn_to_use(head_size, dtype, kv_cache_dtype, block_size,
109 - is_attention_free)
133 + is_attention_free, use_v1)
134 if backend == _Backend.FLASH_ATTN:
135 logger.info("Using Flash Attention backend.")
136 from vllm.attention.backends.flash_attn import ( # noqa: F401
@@ -162,13 +186,12 @@ def get_attn_backend(
186 raise ValueError("Invalid attention backend.")
187
188
165 -def which_attn_to_use(
166 - head_size: int,
167 - dtype: torch.dtype,
168 - kv_cache_dtype: Optional[str],
169 - block_size: int,
170 - is_attention_free: bool,
171 -) -> _Backend:
189 +def which_attn_to_use(head_size: int,
190 + dtype: torch.dtype,
191 + kv_cache_dtype: Optional[str],
192 + block_size: int,
193 + is_attention_free: bool,
194 + use_v1: bool = False) -> _Backend:
195 """Returns which flash attention backend to use."""
196 # Default case.
197 selected_backend = _Backend.FLASH_ATTN
@@ -228,7 +251,7 @@ def which_attn_to_use(
251 if current_platform.is_hpu():
252 return _Backend.HPU_ATTN
253
231 - if envs.VLLM_USE_V1:
254 + if use_v1:
255 return _Backend.FLASH_ATTN_VLLM_V1
256
257 # FlashAttn in NVIDIA GPUs.
vllm/engine/multiprocessing/engine.py
+10 -8
@@ -6,7 +6,9 @@ from typing import Iterator, List, Optional, Union
6 import cloudpickle
7 import zmq
8
9 +import vllm.envs
10 from vllm import AsyncEngineArgs, SamplingParams
11 +from vllm.engine.llm_engine import LLMEngine
12 # yapf conflicts with isort for this block
13 # yapf: disable
14 from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
@@ -17,17 +19,11 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
19 RPCStartupRequest, RPCStartupResponse,
20 RPCUProfileRequest)
21 # yapf: enable
20 -from vllm.envs import VLLM_USE_V1
22 from vllm.executor.gpu_executor import GPUExecutor
23 from vllm.logger import init_logger
24 from vllm.outputs import RequestOutput
25 from vllm.usage.usage_lib import UsageContext
26
26 -if VLLM_USE_V1:
27 - from vllm.v1.engine.llm_engine import LLMEngine
28 -else:
29 - from vllm.engine.llm_engine import LLMEngine
30 -
27 logger = init_logger(__name__)
28
29 POLLING_TIMEOUT_MS = 10000
@@ -117,11 +113,17 @@ class MQLLMEngine:
113 load_general_plugins()
114
115 engine_config = engine_args.create_engine_config()
116 + if vllm.envs.VLLM_USE_V1:
117 + # Lazy import: the v1 package isn't distributed
118 + from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
119 + engine_class = V1LLMEngine
120 + else:
121 + engine_class = LLMEngine
122
121 - executor_class = LLMEngine._get_executor_cls(engine_config)
123 + executor_class = engine_class._get_executor_cls(engine_config)
124
125 use_async_sockets = (engine_config.model_config.use_async_output_proc
124 - and not VLLM_USE_V1)
126 + and not vllm.envs.VLLM_USE_V1)
127
128 return cls(ipc_path=ipc_path,
129 use_async_sockets=use_async_sockets,
vllm/entrypoints/llm.py
+17 -9
@@ -1,7 +1,7 @@
1 import itertools
2 import warnings
3 from contextlib import contextmanager
4 -from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
4 +from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
5 Union, cast, overload)
6
7 from tqdm import tqdm
@@ -10,6 +10,7 @@ from vllm import envs
10 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
11 BeamSearchSequence, get_beam_search_score)
12 from vllm.engine.arg_utils import EngineArgs, TaskOption
13 +from vllm.engine.llm_engine import LLMEngine
14 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
15 apply_hf_chat_template,
16 apply_mistral_chat_template,
@@ -31,11 +32,6 @@ from vllm.transformers_utils.tokenizer_group import TokenizerGroup
32 from vllm.usage.usage_lib import UsageContext
33 from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
34
34 -if envs.VLLM_USE_V1:
35 - from vllm.v1.engine.llm_engine import LLMEngine # type: ignore
36 -else:
37 - from vllm.engine.llm_engine import LLMEngine # type: ignore
38 -
35 logger = init_logger(__name__)
36
37
@@ -206,10 +202,21 @@ class LLM:
202 pooling_returned_token_ids=pooling_returned_token_ids,
203 **kwargs,
204 )
209 - self.llm_engine = LLMEngine.from_engine_args(
205 + # Logic to switch between engines is done at runtime instead of import
206 + # to avoid import order issues
207 + self.engine_class = self.get_engine_class()
208 + self.llm_engine = self.engine_class.from_engine_args(
209 engine_args, usage_context=UsageContext.LLM_CLASS)
210 self.request_counter = Counter()
211
212 + @staticmethod
213 + def get_engine_class() -> Type[LLMEngine]:
214 + if envs.VLLM_USE_V1:
215 + # Lazy import: the v1 package isn't distributed
216 + from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
217 + return V1LLMEngine # type: ignore
218 + return LLMEngine
219 +
220 def get_tokenizer(self) -> AnyTokenizer:
221 return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer
222
@@ -394,7 +401,7 @@ class LLM:
401 priority=priority)
402
403 outputs = self._run_engine(use_tqdm=use_tqdm)
397 - return LLMEngine.validate_outputs(outputs, RequestOutput)
404 + return self.engine_class.validate_outputs(outputs, RequestOutput)
405
406 def beam_search(
407 self,
@@ -769,7 +776,8 @@ class LLM:
776 )
777
778 outputs = self._run_engine(use_tqdm=use_tqdm)
772 - return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
779 + return self.engine_class.validate_outputs(outputs,
780 + EmbeddingRequestOutput)
781
782 def start_profile(self) -> None:
783 self.llm_engine.start_profile()
vllm/model_executor/layers/sampler.py
+9
@@ -30,6 +30,15 @@ if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
30 else:
31 flashinfer_top_k_top_p_sampling = None
32
33 +
34 +def get_sampler() -> torch.nn.Module:
35 + if envs.VLLM_USE_V1:
36 + # Lazy import: the v1 package isn't distributed
37 + from vllm.v1.sample.sampler import Sampler as V1Sampler
38 + return V1Sampler()
39 + return Sampler()
40 +
41 +
42 # (num_token_ids, num_parent_ids) per sequence group.
43 SampleResultType = List[Tuple[List[int], List[int]]]
44
vllm/model_executor/models/arctic.py
+2 -2
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.quantization.deepspeedfp import (
24 DeepSpeedFPConfig, DeepSpeedFPParameter)
25 from vllm.model_executor.layers.rotary_embedding import get_rope
26 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
27 from vllm.model_executor.layers.vocab_parallel_embedding import (
28 ParallelLMHead, VocabParallelEmbedding)
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -436,7 +436,7 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
436 self.unpadded_vocab_size = config.vocab_size
437 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
438 config.vocab_size)
439 - self.sampler = Sampler()
439 + self.sampler = get_sampler()
440 self.make_empty_intermediate_tensors = (
441 self.model.make_empty_intermediate_tensors)
442
vllm/model_executor/models/baichuan.py
+2 -2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization import QuantizationConfig
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -352,7 +352,7 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
352 if self.config.tie_word_embeddings:
353 self.lm_head.weight = self.model.embed_tokens.weight
354 self.logits_processor = LogitsProcessor(config.vocab_size)
355 - self.sampler = Sampler()
355 + self.sampler = get_sampler()
356 self.make_empty_intermediate_tensors = (
357 self.model.make_empty_intermediate_tensors)
358
vllm/model_executor/models/bart.py
+2 -2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -838,7 +838,7 @@ class BartForConditionalGeneration(nn.Module):
838
839 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
840 config.vocab_size)
841 - self.sampler = Sampler()
841 + self.sampler = get_sampler()
842
843 def forward(
844 self,
vllm/model_executor/models/blip2.py
+2 -2
@@ -13,7 +13,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13 InputContext, token_inputs)
14 from vllm.model_executor.layers.activation import get_act_fn
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 from vllm.multimodal.utils import consecutive_placeholder_ranges
@@ -525,7 +525,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
525 if hasattr(self.language_model, "sampler"):
526 return self.language_model.sampler
527
528 - return Sampler()
528 + return get_sampler()
529
530 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
531 h = w = self.config.vision_config.image_size
vllm/model_executor/models/bloom.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 RowParallelLinear)
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization import QuantizationConfig
36 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -298,7 +298,7 @@ class BloomForCausalLM(nn.Module, SupportsPP):
298 self.config.hidden_size)
299
300 self.logits_processor = LogitsProcessor(config.vocab_size)
301 - self.sampler = Sampler()
301 + self.sampler = get_sampler()
302 self.make_empty_intermediate_tensors = (
303 self.transformer.make_empty_intermediate_tensors)
304
vllm/model_executor/models/chameleon.py
+2 -2
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
21 from vllm.model_executor.layers.logits_processor import LogitsProcessor
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.rotary_embedding import get_rope
24 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 from vllm.model_executor.layers.vocab_parallel_embedding import (
26 ParallelLMHead, VocabParallelEmbedding)
27 from vllm.model_executor.model_loader.weight_utils import (
@@ -946,7 +946,7 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
946 logit_scale = getattr(config, "logit_scale", 1.0)
947 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
948 config.vocab_size, logit_scale)
949 - self.sampler = Sampler()
949 + self.sampler = get_sampler()
950 self.make_empty_intermediate_tensors = (
951 self.model.make_empty_intermediate_tensors)
952
vllm/model_executor/models/chatglm.py
+2 -2
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
24 from vllm.model_executor.layers.logits_processor import LogitsProcessor
25 from vllm.model_executor.layers.quantization import QuantizationConfig
26 from vllm.model_executor.layers.rotary_embedding import get_rope
27 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
27 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
28 from vllm.model_executor.layers.vocab_parallel_embedding import (
29 ParallelLMHead, VocabParallelEmbedding)
30 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -616,7 +616,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
616 self.transformer.embedding.weight)
617 self.lm_head = self.transformer.output_layer
618 self.logits_processor = LogitsProcessor(config.padded_vocab_size)
619 - self.sampler = Sampler()
619 + self.sampler = get_sampler()
620
621 def forward(self,
622 input_ids: torch.Tensor,
vllm/model_executor/models/commandr.py
+2 -2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization import QuantizationConfig
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 VocabParallelEmbedding)
43 from vllm.model_executor.model_loader.weight_utils import (
@@ -355,7 +355,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
355 cache_config,
356 quant_config,
357 lora_config=lora_config)
358 - self.sampler = Sampler()
358 + self.sampler = get_sampler()
359 self.make_empty_intermediate_tensors = (
360 self.model.make_empty_intermediate_tensors)
361
vllm/model_executor/models/dbrx.py
+2 -2
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
14 from vllm.model_executor.layers.logits_processor import LogitsProcessor
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.layers.rotary_embedding import get_rope
17 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
19 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
20 from vllm.model_executor.model_loader.weight_utils import (
@@ -373,7 +373,7 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
373 )
374 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
375 config.vocab_size)
376 - self.sampler = Sampler()
376 + self.sampler = get_sampler()
377 self.make_empty_intermediate_tensors = (
378 self.transformer.make_empty_intermediate_tensors)
379
vllm/model_executor/models/deepseek.py
+2 -2
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.quantization import QuantizationConfig
43 from vllm.model_executor.layers.rotary_embedding import get_rope
44 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
44 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45 from vllm.model_executor.layers.vocab_parallel_embedding import (
46 ParallelLMHead, VocabParallelEmbedding)
47 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -399,7 +399,7 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
399 if self.config.tie_word_embeddings:
400 self.lm_head.weight = self.model.embed_tokens.weight
401 self.logits_processor = LogitsProcessor(config.vocab_size)
402 - self.sampler = Sampler()
402 + self.sampler = get_sampler()
403 self.make_empty_intermediate_tensors = (
404 self.model.make_empty_intermediate_tensors)
405
vllm/model_executor/models/deepseek_v2.py
+2 -2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
42 from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 from vllm.model_executor.layers.quantization import QuantizationConfig
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -496,7 +496,7 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
496 config.hidden_size,
497 quant_config=quant_config)
498 self.logits_processor = LogitsProcessor(config.vocab_size)
499 - self.sampler = Sampler()
499 + self.sampler = get_sampler()
500 self.make_empty_intermediate_tensors = (
501 self.model.make_empty_intermediate_tensors)
502
vllm/model_executor/models/exaone.py
+2 -2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
42 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 get_compressed_tensors_cache_scale)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import (
@@ -478,7 +478,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
478 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
479 config.vocab_size,
480 logit_scale)
481 - self.sampler = Sampler()
481 + self.sampler = get_sampler()
482 else:
483 self.lm_head = PPMissingLayer()
484
vllm/model_executor/models/falcon.py
+2 -2
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
38 from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 from vllm.model_executor.layers.quantization import QuantizationConfig
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -426,7 +426,7 @@ class FalconForCausalLM(nn.Module, SupportsPP):
426 quant_config=quant_config,
427 )
428 self.logits_processor = LogitsProcessor(config.vocab_size)
429 - self.sampler = Sampler()
429 + self.sampler = get_sampler()
430 self.make_empty_intermediate_tensors = (
431 self.transformer.make_empty_intermediate_tensors)
432
vllm/model_executor/models/florence2.py
+2 -2
@@ -10,7 +10,7 @@ from vllm.config import CacheConfig
10 from vllm.model_executor.layers.logits_processor import LogitsProcessor
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
13 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
14 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
15 from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
16 BartParallelLMHead,
@@ -112,7 +112,7 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
112
113 self.logits_processor = LogitsProcessor(self.vocab_size,
114 config.vocab_size)
115 - self.sampler = Sampler()
115 + self.sampler = get_sampler()
116
117 def forward(
118 self,
vllm/model_executor/models/gemma.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.quantization import QuantizationConfig
35 from vllm.model_executor.layers.rotary_embedding import get_rope
36 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -393,7 +393,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
393 quant_config,
394 prefix=maybe_prefix(prefix, "model"))
395 self.logits_processor = LogitsProcessor(config.vocab_size)
396 - self.sampler = Sampler()
396 + self.sampler = get_sampler()
397 self.make_empty_intermediate_tensors = (
398 self.model.make_empty_intermediate_tensors)
399
vllm/model_executor/models/gemma2.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.pooler import Pooler, PoolingType
34 from vllm.model_executor.layers.quantization import QuantizationConfig
35 from vllm.model_executor.layers.rotary_embedding import get_rope
36 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -414,7 +414,7 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
414 self.model = Gemma2Model(config, cache_config, quant_config)
415 self.logits_processor = LogitsProcessor(
416 config.vocab_size, soft_cap=config.final_logit_softcapping)
417 - self.sampler = Sampler()
417 + self.sampler = get_sampler()
418 self.make_empty_intermediate_tensors = (
419 self.model.make_empty_intermediate_tensors)
420
vllm/model_executor/models/gpt2.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 RowParallelLinear)
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization import QuantizationConfig
36 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -259,7 +259,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
259 self.lm_head = ParallelLMHead(self.config.vocab_size,
260 self.config.hidden_size)
261 self.logits_processor = LogitsProcessor(config.vocab_size)
262 - self.sampler = Sampler()
262 + self.sampler = get_sampler()
263 self.make_empty_intermediate_tensors = (
264 self.transformer.make_empty_intermediate_tensors)
265
vllm/model_executor/models/gpt_bigcode.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 RowParallelLinear)
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization import QuantizationConfig
36 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -285,7 +285,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
285 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
286 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
287 config.vocab_size)
288 - self.sampler = Sampler()
288 + self.sampler = get_sampler()
289 self.make_empty_intermediate_tensors = (
290 self.transformer.make_empty_intermediate_tensors)
291
vllm/model_executor/models/gpt_j.py
+2 -2
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 ParallelLMHead, VocabParallelEmbedding)
38 from vllm.model_executor.model_loader.weight_utils import (
@@ -247,7 +247,7 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
247 quant_config=quant_config,
248 )
249 self.logits_processor = LogitsProcessor(config.vocab_size)
250 - self.sampler = Sampler()
250 + self.sampler = get_sampler()
251 self.make_empty_intermediate_tensors = (
252 self.transformer.make_empty_intermediate_tensors)
253
vllm/model_executor/models/gpt_neox.py
+2 -2
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 ParallelLMHead, VocabParallelEmbedding)
38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -260,7 +260,7 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
260 if self.config.tie_word_embeddings:
261 self.embed_out.weight = self.gpt_neox.embed_in.weight
262 self.logits_processor = LogitsProcessor(config.vocab_size)
263 - self.sampler = Sampler()
263 + self.sampler = get_sampler()
264 self.make_empty_intermediate_tensors = (
265 self.gpt_neox.make_empty_intermediate_tensors)
266
vllm/model_executor/models/granite.py
+2 -2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization.base_config import (
42 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 get_compressed_tensors_cache_scale)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import (
@@ -411,7 +411,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
412 config.vocab_size,
413 scale=logit_scale)
414 - self.sampler = Sampler()
414 + self.sampler = get_sampler()
415 else:
416 self.lm_head = PPMissingLayer()
417
vllm/model_executor/models/granitemoe.py
+2 -2
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 from vllm.model_executor.layers.quantization.base_config import (
40 QuantizationConfig)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -371,7 +371,7 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
371 scale=1 /
372 self.config.logits_scaling)
373
374 - self.sampler = Sampler()
374 + self.sampler = get_sampler()
375
376 def forward(
377 self,
vllm/model_executor/models/internlm2.py
+2 -2
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
20 from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.rotary_embedding import get_rope
23 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 ParallelLMHead, VocabParallelEmbedding)
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -338,7 +338,7 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
338 if self.config.tie_word_embeddings:
339 self.output.weight = self.model.tok_embeddings.weight
340 self.logits_processor = LogitsProcessor(config.vocab_size)
341 - self.sampler = Sampler()
341 + self.sampler = get_sampler()
342 self.make_empty_intermediate_tensors = (
343 self.model.make_empty_intermediate_tensors)
344
vllm/model_executor/models/internvl.py
+2 -2
@@ -21,7 +21,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 InputContext, token_inputs)
22 from vllm.model_executor.layers.quantization import (AWQConfig,
23 QuantizationConfig)
24 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -467,7 +467,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
467 if hasattr(self.language_model, "sampler"):
468 return self.language_model.sampler
469
470 - return Sampler()
470 + return get_sampler()
471
472 def _init_vision_model(
473 self,
vllm/model_executor/models/jais.py
+2 -2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 RowParallelLinear)
35 from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.quantization import QuantizationConfig
37 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -308,7 +308,7 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
308 config.mup_width_scale)
309 self.logits_processor = LogitsProcessor(vocab_size=config.vocab_size,
310 scale=self.output_logits_scale)
311 - self.sampler = Sampler()
311 + self.sampler = get_sampler()
312 self.make_empty_intermediate_tensors = (
313 self.transformer.make_empty_intermediate_tensors)
314
vllm/model_executor/models/jamba.py
+2 -2
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
17 from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
19 from vllm.model_executor.layers.quantization import QuantizationConfig
20 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
20 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21 from vllm.model_executor.layers.vocab_parallel_embedding import (
22 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -383,7 +383,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
383
384 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
385 config.vocab_size)
386 - self.sampler = Sampler()
386 + self.sampler = get_sampler()
387
388 def forward(self,
389 input_ids: torch.Tensor,
vllm/model_executor/models/llama.py
+2 -2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
42 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 get_compressed_tensors_cache_scale)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import (
@@ -536,7 +536,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
536 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
537 config.vocab_size,
538 logit_scale)
539 - self.sampler = Sampler()
539 + self.sampler = get_sampler()
540 else:
541 self.lm_head = PPMissingLayer()
542 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/llava.py
+2 -2
@@ -14,7 +14,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14 InputContext)
15 from vllm.model_executor.layers.activation import get_act_fn
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 from vllm.sequence import IntermediateTensors
@@ -302,7 +302,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
302 if hasattr(self.language_model, "sampler"):
303 return self.language_model.sampler
304
305 - return Sampler()
305 + return get_sampler()
306
307 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
308 h = w = self.config.vision_config.image_size
vllm/model_executor/models/llava_next.py
+2 -2
@@ -16,7 +16,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16 InputContext)
17 from vllm.model_executor.layers.pooler import Pooler, PoolingType
18 from vllm.model_executor.layers.quantization import QuantizationConfig
19 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20 from vllm.model_executor.pooling_metadata import PoolingMetadata
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -327,7 +327,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
327 if hasattr(self.language_model, "sampler"):
328 return self.language_model.sampler
329
330 - return Sampler()
330 + return get_sampler()
331
332 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
333 expected_dims = (2, )
vllm/model_executor/models/llava_next_video.py
+2 -2
@@ -15,7 +15,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15 InputContext, token_inputs)
16 from vllm.model_executor.layers.activation import get_act_fn
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19 from vllm.model_executor.models.clip import CLIPVisionModel
20 from vllm.model_executor.sampling_metadata import SamplingMetadata
21 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -289,7 +289,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
289 if hasattr(self.language_model, "sampler"):
290 return self.language_model.sampler
291
292 - return Sampler()
292 + return get_sampler()
293
294 def _validate_video_pixel_values(
295 self, data: Union[torch.Tensor, List[torch.Tensor]]
vllm/model_executor/models/llava_onevision.py
+2 -2
@@ -19,7 +19,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19 InputContext, token_inputs)
20 from vllm.model_executor.layers.activation import get_act_fn
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
22 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.multimodal.utils import (cached_get_tokenizer,
@@ -437,7 +437,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
437 if hasattr(self.language_model, "sampler"):
438 return self.language_model.sampler
439
440 - return Sampler()
440 + return get_sampler()
441
442 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
443 expected_dims = (2, )
vllm/model_executor/models/mamba.py
+2 -2
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
14 from vllm.model_executor.layers.quantization.base_config import (
15 QuantizationConfig)
16 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17 from vllm.model_executor.layers.vocab_parallel_embedding import (
18 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
19 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -169,7 +169,7 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
169
170 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
171 config.vocab_size)
172 - self.sampler = Sampler()
172 + self.sampler = get_sampler()
173
174 def forward(self,
175 input_ids: torch.Tensor,
vllm/model_executor/models/minicpm.py
+2 -2
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
43 from vllm.model_executor.layers.logits_processor import LogitsProcessor
44 from vllm.model_executor.layers.quantization import QuantizationConfig
45 from vllm.model_executor.layers.rotary_embedding import get_rope
46 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
46 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
47 from vllm.model_executor.layers.vocab_parallel_embedding import (
48 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
49 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -496,7 +496,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
496
497 self.logits_processor = LogitsProcessor(unpadded_vocab_size,
498 config.vocab_size)
499 - self.sampler = Sampler()
499 + self.sampler = get_sampler()
500 self.make_empty_intermediate_tensors = (
501 self.model.make_empty_intermediate_tensors)
502
vllm/model_executor/models/minicpmv.py
+2 -2
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.quantization import QuantizationConfig
42 from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
43 get_2d_sincos_pos_embed)
44 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
44 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
46 from vllm.model_executor.model_loader.utils import set_default_torch_dtype
47 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -420,7 +420,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
420 quant_config=quant_config,
421 prefix="llm.lm_head")
422 self.logits_processor = LogitsProcessor(config.vocab_size)
423 - self.sampler = Sampler()
423 + self.sampler = get_sampler()
424
425 self.make_empty_intermediate_tensors = (
426 self.llm.make_empty_intermediate_tensors)
vllm/model_executor/models/mixtral.py
+2 -2
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
38 from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 from vllm.model_executor.layers.quantization import QuantizationConfig
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import (
@@ -366,7 +366,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
366 self.lm_head.weight = self.model.embed_tokens.weight
367 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
368 config.vocab_size)
369 - self.sampler = Sampler()
369 + self.sampler = get_sampler()
370 self.make_empty_intermediate_tensors = (
371 self.model.make_empty_intermediate_tensors)
372
vllm/model_executor/models/mixtral_quant.py
+2 -2
@@ -40,7 +40,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
40 from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.quantization import QuantizationConfig
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 ParallelLMHead, VocabParallelEmbedding)
46 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -366,7 +366,7 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
366 if self.config.tie_word_embeddings:
367 self.lm_head.weight = self.model.embed_tokens.weight
368 self.logits_processor = LogitsProcessor(config.vocab_size)
369 - self.sampler = Sampler()
369 + self.sampler = get_sampler()
370 self.make_empty_intermediate_tensors = (
371 self.model.make_empty_intermediate_tensors)
372
vllm/model_executor/models/mllama.py
+2 -2
@@ -44,7 +44,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
44 RowParallelLinear)
45 from vllm.model_executor.layers.logits_processor import LogitsProcessor
46 from vllm.model_executor.layers.quantization import QuantizationConfig
47 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
48 from vllm.model_executor.layers.vocab_parallel_embedding import (
49 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -1141,7 +1141,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1141 )
1142 self.logits_processor = LogitsProcessor(config.output_hidden_states,
1143 config.text_config.vocab_size)
1144 - self.sampler = Sampler()
1144 + self.sampler = get_sampler()
1145
1146 def compute_logits(
1147 self,
vllm/model_executor/models/mlp_speculator.py
+2 -2
@@ -6,7 +6,7 @@ import torch.nn as nn
6
7 from vllm.model_executor import SamplingMetadata
8 from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
9 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
10 from vllm.model_executor.layers.vocab_parallel_embedding import (
11 ParallelLMHead, VocabParallelEmbedding)
12 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -137,7 +137,7 @@ class MLPSpeculator(nn.Module):
137 self.config = config
138 self.logits_processor = LogitsProcessor(config.vocab_size,
139 config.vocab_size, 1.0)
140 - self.sampler = Sampler()
140 + self.sampler = get_sampler()
141
142 def generate_proposals(
143 self,
vllm/model_executor/models/molmo.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.quantization import QuantizationConfig
35 from vllm.model_executor.layers.rotary_embedding import get_rope
36 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -1053,7 +1053,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1053
1054 self.logits_processor = LogitsProcessor(config.embedding_size
1055 or config.vocab_size)
1056 - self.sampler = Sampler()
1056 + self.sampler = get_sampler()
1057
1058 self.make_empty_intermediate_tensors = (
1059 self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/mpt.py
+2 -2
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 RowParallelLinear)
17 from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.quantization import QuantizationConfig
19 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 VocabParallelEmbedding)
22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -281,7 +281,7 @@ class MPTForCausalLM(nn.Module, SupportsPP):
281 self.transformer = MPTModel(config, cache_config, quant_config)
282 self.lm_head = self.transformer.wte
283 self.logits_processor = LogitsProcessor(config.vocab_size)
284 - self.sampler = Sampler()
284 + self.sampler = get_sampler()
285 self.make_empty_intermediate_tensors = (
286 self.transformer.make_empty_intermediate_tensors)
287
vllm/model_executor/models/nemotron.py
+2 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
36 from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 from vllm.model_executor.layers.quantization import QuantizationConfig
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import (
@@ -441,7 +441,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
441 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
442 config.vocab_size,
443 logit_scale)
444 - self.sampler = Sampler()
444 + self.sampler = get_sampler()
445 else:
446 self.lm_head = PPMissingLayer()
447 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/olmo.py
+2 -2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization import QuantizationConfig
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -309,7 +309,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
309 quant_config=quant_config,
310 )
311 self.logits_processor = LogitsProcessor(config.vocab_size)
312 - self.sampler = Sampler()
312 + self.sampler = get_sampler()
313 self.make_empty_intermediate_tensors = (
314 self.model.make_empty_intermediate_tensors)
315
vllm/model_executor/models/olmoe.py
+2 -2
@@ -28,7 +28,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
28 from vllm.model_executor.layers.logits_processor import LogitsProcessor
29 from vllm.model_executor.layers.quantization import QuantizationConfig
30 from vllm.model_executor.layers.rotary_embedding import get_rope
31 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
31 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
32 from vllm.model_executor.layers.vocab_parallel_embedding import (
33 ParallelLMHead, VocabParallelEmbedding)
34 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -323,7 +323,7 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
323 config.hidden_size,
324 quant_config=quant_config)
325 self.logits_processor = LogitsProcessor(config.vocab_size)
326 - self.sampler = Sampler()
326 + self.sampler = get_sampler()
327
328 self.make_empty_intermediate_tensors = (
329 self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/opt.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 RowParallelLinear)
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization import QuantizationConfig
36 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -362,7 +362,7 @@ class OPTForCausalLM(nn.Module, SupportsPP):
362 self.lm_head = ParallelLMHead(config.vocab_size,
363 config.word_embed_proj_dim)
364 self.logits_processor = LogitsProcessor(config.vocab_size)
365 - self.sampler = Sampler()
365 + self.sampler = get_sampler()
366 self.make_empty_intermediate_tensors = (
367 self.model.make_empty_intermediate_tensors)
368
vllm/model_executor/models/orion.py
+2 -2
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
20 from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.rotary_embedding import get_rope
23 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 ParallelLMHead, VocabParallelEmbedding)
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -284,7 +284,7 @@ class OrionForCausalLM(nn.Module, SupportsPP):
284 if self.config.tie_word_embeddings:
285 self.lm_head.weight = self.model.embed_tokens.weight
286 self.logits_processor = LogitsProcessor(config.vocab_size)
287 - self.sampler = Sampler()
287 + self.sampler = get_sampler()
288 self.make_empty_intermediate_tensors = (
289 self.model.make_empty_intermediate_tensors)
290
vllm/model_executor/models/persimmon.py
+2 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
36 from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 from vllm.model_executor.layers.quantization import QuantizationConfig
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -279,7 +279,7 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
279 config.hidden_size,
280 bias=False)
281 self.logits_processor = LogitsProcessor(config.vocab_size)
282 - self.sampler = Sampler()
282 + self.sampler = get_sampler()
283 self.make_empty_intermediate_tensors = (
284 self.model.make_empty_intermediate_tensors)
285
vllm/model_executor/models/phi.py
+2 -2
@@ -51,7 +51,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51 from vllm.model_executor.layers.logits_processor import LogitsProcessor
52 from vllm.model_executor.layers.quantization import QuantizationConfig
53 from vllm.model_executor.layers.rotary_embedding import get_rope
54 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
54 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55 from vllm.model_executor.layers.vocab_parallel_embedding import (
56 ParallelLMHead, VocabParallelEmbedding)
57 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -300,7 +300,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
300 bias=True,
301 quant_config=quant_config)
302 self.logits_processor = LogitsProcessor(config.vocab_size)
303 - self.sampler = Sampler()
303 + self.sampler = get_sampler()
304 self.make_empty_intermediate_tensors = (
305 self.model.make_empty_intermediate_tensors)
306
vllm/model_executor/models/phi3_small.py
+2 -2
@@ -15,7 +15,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
15 from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.rotary_embedding import get_rope
18 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
21 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -386,7 +386,7 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
386 if self.config.tie_word_embeddings:
387 self.lm_head.weight = self.model.embed_tokens.weight
388 self.logits_processor = LogitsProcessor(config.vocab_size)
389 - self.sampler = Sampler()
389 + self.sampler = get_sampler()
390 self.make_empty_intermediate_tensors = (
391 self.model.make_empty_intermediate_tensors)
392
vllm/model_executor/models/phi3v.py
+2 -2
@@ -32,7 +32,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
32 from vllm.logger import init_logger
33 from vllm.model_executor.layers.pooler import Pooler, PoolingType
34 from vllm.model_executor.layers.quantization import QuantizationConfig
35 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
38 from vllm.model_executor.models.clip import CLIPVisionModel
@@ -570,7 +570,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
570 if hasattr(self.language_model, "sampler"):
571 return self.language_model.sampler
572
573 - return Sampler()
573 + return get_sampler()
574
575 def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
576 expected_dims = (2, )
vllm/model_executor/models/phimoe.py
+2 -2
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization.base_config import (
39 QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import (
@@ -562,7 +562,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
562 )
563 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
564 config.vocab_size)
565 - self.sampler = Sampler()
565 + self.sampler = get_sampler()
566
567 self.make_empty_intermediate_tensors = (
568 self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/pixtral.py
+2 -2
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
25 QKVParallelLinear,
26 RowParallelLinear)
27 from vllm.model_executor.layers.quantization import QuantizationConfig
28 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
28 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 from vllm.model_executor.models.utils import merge_multimodal_embeddings
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -190,7 +190,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
190 if hasattr(self.language_model, "sampler"):
191 return self.language_model.sampler
192
193 - return Sampler()
193 + return get_sampler()
194
195 def forward(
196 self,
vllm/model_executor/models/qwen.py
+2 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.quantization import QuantizationConfig
37 from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -884,7 +884,7 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
884 if self.config.tie_word_embeddings:
885 self.lm_head.weight = self.transformer.wte.weight
886 self.logits_processor = LogitsProcessor(config.vocab_size)
887 - self.sampler = Sampler()
887 + self.sampler = get_sampler()
888 self.make_empty_intermediate_tensors = (
889 self.transformer.make_empty_intermediate_tensors)
890
vllm/model_executor/models/qwen2.py
+2 -2
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 from vllm.model_executor.layers.quantization import QuantizationConfig
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import (
@@ -444,7 +444,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
444 prefix, "lm_head"))
445
446 self.logits_processor = LogitsProcessor(config.vocab_size)
447 - self.sampler = Sampler()
447 + self.sampler = get_sampler()
448 self.make_empty_intermediate_tensors = (
449 self.model.make_empty_intermediate_tensors)
450
vllm/model_executor/models/qwen2_audio.py
+2 -2
@@ -36,7 +36,7 @@ from vllm.logger import init_logger
36 from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 from vllm.model_executor.layers.quantization.base_config import (
38 QuantizationConfig)
39 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
41 from vllm.model_executor.model_loader.weight_utils import (
42 default_weight_loader, maybe_remap_kv_scale_name)
@@ -295,7 +295,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
295 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
296 config.text_config.vocab_size,
297 logit_scale)
298 - self.sampler = Sampler()
298 + self.sampler = get_sampler()
299
300 self.make_empty_intermediate_tensors = (
301 self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/qwen2_moe.py
+2 -2
@@ -44,7 +44,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
44 from vllm.model_executor.layers.logits_processor import LogitsProcessor
45 from vllm.model_executor.layers.quantization import QuantizationConfig
46 from vllm.model_executor.layers.rotary_embedding import get_rope
47 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
48 from vllm.model_executor.layers.vocab_parallel_embedding import (
49 ParallelLMHead, VocabParallelEmbedding)
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -393,7 +393,7 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
393 if self.config.tie_word_embeddings:
394 self.lm_head.weight = self.model.embed_tokens.weight
395 self.logits_processor = LogitsProcessor(config.vocab_size)
396 - self.sampler = Sampler()
396 + self.sampler = get_sampler()
397 self.make_empty_intermediate_tensors = (
398 self.model.make_empty_intermediate_tensors)
399
vllm/model_executor/models/qwen2_vl.py
+2 -2
@@ -52,7 +52,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52 RowParallelLinear)
53 from vllm.model_executor.layers.logits_processor import LogitsProcessor
54 from vllm.model_executor.layers.quantization import QuantizationConfig
55 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
55 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
56 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
57 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
58 from vllm.model_executor.models.qwen2 import Qwen2Model
@@ -990,7 +990,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
990 self.lm_head = PPMissingLayer()
991
992 self.logits_processor = LogitsProcessor(config.vocab_size)
993 - self.sampler = Sampler()
993 + self.sampler = get_sampler()
994 self.make_empty_intermediate_tensors = (
995 make_empty_intermediate_tensors_factory(
996 ["hidden_states", "residual"], config.hidden_size))
vllm/model_executor/models/solar.py
+2 -2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
42 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 get_compressed_tensors_cache_scale)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import (
@@ -449,7 +449,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
449 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
450 config.vocab_size,
451 logit_scale)
452 - self.sampler = Sampler()
452 + self.sampler = get_sampler()
453 else:
454 self.lm_head = PPMissingLayer()
455
vllm/model_executor/models/stablelm.py
+2 -2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization import QuantizationConfig
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -261,7 +261,7 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
261 if self.config.tie_word_embeddings:
262 self.lm_head.weight = self.model.embed_tokens.weight
263 self.logits_processor = LogitsProcessor(config.vocab_size)
264 - self.sampler = Sampler()
264 + self.sampler = get_sampler()
265 self.make_empty_intermediate_tensors = (
266 self.model.make_empty_intermediate_tensors)
267
vllm/model_executor/models/starcoder2.py
+2 -2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization import QuantizationConfig
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -269,7 +269,7 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
269 )
270 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
271 config.vocab_size)
272 - self.sampler = Sampler()
272 + self.sampler = get_sampler()
273 self.make_empty_intermediate_tensors = (
274 self.model.make_empty_intermediate_tensors)
275
vllm/model_executor/models/ultravox.py
+2 -2
@@ -21,7 +21,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
22 from vllm.model_executor.layers.layernorm import RMSNorm
23 from vllm.model_executor.layers.quantization import QuantizationConfig
24 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 from vllm.model_executor.model_loader.loader import DefaultModelLoader
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
@@ -379,7 +379,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
379 if hasattr(self.language_model, "sampler"):
380 return self.language_model.sampler
381
382 - return Sampler()
382 + return get_sampler()
383
384 def _audio_features_to_embeddings(
385 self, input_features: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/xverse.py
+2 -2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization import QuantizationConfig
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 -from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -334,7 +334,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
334 if self.config.tie_word_embeddings:
335 self.lm_head.weight = self.model.embed_tokens.weight
336 self.logits_processor = LogitsProcessor(config.vocab_size)
337 - self.sampler = Sampler()
337 + self.sampler = get_sampler()
338 self.make_empty_intermediate_tensors = (
339 self.model.make_empty_intermediate_tensors)
340
vllm/v1/attention/backends/flash_attn.py
+6 -6
@@ -136,7 +136,7 @@ class FlashAttentionImpl(AttentionImpl):
136 "key/v_scale is not supported in FlashAttention.")
137
138 output = torch.empty_like(query)
139 - torch.ops.vllm.unified_flash_attention(
139 + torch.ops.vllm.unified_v1_flash_attention(
140 output,
141 query,
142 key,
@@ -156,7 +156,7 @@ class FlashAttentionImpl(AttentionImpl):
156 return output
157
158
159 -def unified_flash_attention(
159 +def unified_v1_flash_attention(
160 output: torch.Tensor,
161 query: torch.Tensor,
162 key: torch.Tensor,
@@ -222,7 +222,7 @@ def unified_flash_attention(
222 output[:num_actual_tokens].copy_(attn_output)
223
224
225 -def unified_flash_attention_fake(
225 +def unified_v1_flash_attention_fake(
226 output: torch.Tensor,
227 query: torch.Tensor,
228 key: torch.Tensor,
@@ -243,8 +243,8 @@ def unified_flash_attention_fake(
243
244
245 direct_register_custom_op(
246 - op_name="unified_flash_attention",
247 - op_func=unified_flash_attention,
246 + op_name="unified_v1_flash_attention",
247 + op_func=unified_v1_flash_attention,
248 mutates_args=["kv_cache", "output"],
249 - fake_impl=unified_flash_attention_fake,
249 + fake_impl=unified_v1_flash_attention_fake,
250 )
vllm/v1/engine/llm_engine.py
+6
@@ -155,6 +155,12 @@ class LLMEngine:
155 # GPU and CPU blocks, which are profiled in the distributed executor.
156 self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
157
158 + def __del__(self):
159 + # Small hack- implicit clean up of resources on garbage collect
160 + # TODO: this should probably be explicitly invoked when we're done with
161 + # the engine
162 + self.terminate_detokenizer()
163 +
164 def _initialize_kv_caches(self) -> None:
165 num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
166 )
vllm/v1/tokenizer/detokenizer.py
+4 -4
@@ -73,7 +73,7 @@ class Detokenizer:
73 return None
74
75 def terminate(self) -> None:
76 - self.push_socket.send(b"", flags=zmq.NOBLOCK)
76 + self.detokenizer.kill()
77 self.detokenizer.join()
78
79
@@ -108,10 +108,10 @@ class DetokenizerProc(multiprocessing.Process):
108 self.push_socket.bind(f"tcp://*:{self.push_port}")
109
110 while True:
111 + if self.pull_socket.poll(timeout=1000) == 0:
112 + # Nothing to read
113 + continue
114 message = self.pull_socket.recv()
112 - if message == b"":
113 - # Terminate signal.
114 - break
115 inputs = self.msgpack_decoder.decode(message)
116
117 for req_id in inputs.free_req_ids:
vllm/v1/worker/gpu_model_runner.py
+1 -4
@@ -2,7 +2,6 @@ import os
2 import time
3 from dataclasses import dataclass
4 from typing import TYPE_CHECKING, Dict, List, Optional, Set
5 -from unittest.mock import patch
5
6 import numpy as np
7 import torch
@@ -26,7 +25,6 @@ from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
25 FlashAttentionMetadata)
26 from vllm.v1.outputs import ModelRunnerOutput
27 from vllm.v1.sample.metadata import SamplingMetadata
29 -from vllm.v1.sample.sampler import Sampler
28
29 if TYPE_CHECKING:
30 from vllm.v1.core.scheduler import SchedulerOutput
@@ -418,8 +416,7 @@ class GPUModelRunner:
416
417 logger.info("Starting to load model %s...", self.model_config.model)
418 with DeviceMemoryProfiler() as m: # noqa: SIM117
421 - with patch("vllm.model_executor.layers.sampler.Sampler", Sampler):
422 - self.model = get_model(vllm_config=self.vllm_config)
419 + self.model = get_model(vllm_config=self.vllm_config)
420
421 self.model_memory_usage = m.consumed_memory
422 logger.info("Loading model weights took %.4f GB",