A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Make v1 more testable (#9888)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Nov 6, 2024 at 12:57 UTC
d58268c56a8ee0eb01c30e7ab7c07c934e1791c2
75 files changed
+243
-165
Dockerfile
+3
@@ -191,6 +191,9 @@ ADD . /vllm-workspace/
191
RUN --mount=type=cache,target=/root/.cache/pip \
192
python3 -m pip install -r requirements-dev.txt
193
194
+# Copy in the v1 package for testing (it isn't distributed yet)
195
+COPY vllm/v1 /usr/local/lib/python3.12/dist-packages/vllm/v1
196
+
197
# doc requires source code
198
# we hide them inside `test_docs/` , so that this source code
199
# will not be imported by other tests
pyproject.toml
+1
@@ -97,4 +97,5 @@ markers = [
97
"skip_global_cleanup",
98
"core_model: run this model test in each PR instead of just daily",
99
"distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
100
+ "skip_v1: do not run this test with v1",
101
]
tests/conftest.py
+18
@@ -5,6 +5,7 @@ from collections import UserList
5
from enum import Enum
6
from typing import (Any, Callable, Dict, List, Optional, Tuple, Type,
7
TypedDict, TypeVar, Union)
8
+from unittest.mock import patch
9
10
import numpy as np
11
import pytest
@@ -108,6 +109,23 @@ VIDEO_ASSETS = _VideoAssets()
109
"""Singleton instance of :class:`_VideoAssets`."""
110
111
112
+@pytest.fixture(params=[True, False])
113
+def run_with_both_engines(request):
114
+ # Automatically runs tests twice, once with V1 and once without
115
+ use_v1 = request.param
116
+ # Tests decorated with `@skip_v1` are only run without v1
117
+ skip_v1 = request.node.get_closest_marker("skip_v1")
118
+
119
+ if use_v1:
120
+ if skip_v1:
121
+ pytest.skip("Skipping test on vllm V1")
122
+ with patch('vllm.envs.VLLM_USE_V1', True):
123
+ yield
124
+ else:
125
+ with patch('vllm.envs.VLLM_USE_V1', False):
126
+ yield
127
+
128
+
129
@pytest.fixture(autouse=True)
130
def init_test_http_connection():
131
# pytest_asyncio may use a different event loop per test
tests/entrypoints/llm/test_prompt_validation.py
+9
@@ -3,12 +3,21 @@ import pytest
3
from vllm import LLM
4
5
6
+@pytest.fixture(autouse=True)
7
+def v1(run_with_both_engines):
8
+ # Simple autouse wrapper to run both engines for each test
9
+ # This can be promoted up to conftest.py to run for every
10
+ # test in a package
11
+ pass
12
+
13
+
14
def test_empty_prompt():
15
llm = LLM(model="gpt2", enforce_eager=True)
16
with pytest.raises(ValueError, match='Prompt cannot be empty'):
17
llm.generate([""])
18
19
20
+@pytest.mark.skip_v1
21
def test_out_of_vocab_token():
22
llm = LLM(model="gpt2", enforce_eager=True)
23
with pytest.raises(ValueError, match='out of vocabulary'):
tests/kernels/test_attention_selector.py
+2
@@ -44,6 +44,8 @@ def test_env(name: str, device: str, monkeypatch):
44
45
def test_flash_attn(monkeypatch):
46
"""Test FlashAttn validation."""
47
+ # TODO: When testing for v1, pipe in `use_v1` as an argument to
48
+ # which_attn_to_use
49
50
override_backend_env_variable(monkeypatch, STR_FLASH_ATTN_VAL)
51
tests/kernels/test_encoder_decoder_attn.py
+2
-2
@@ -16,7 +16,7 @@ from tests.kernels.utils import *
16
from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
17
AttentionType)
18
from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19
-from vllm.attention.selector import (_Backend, get_attn_backend,
19
+from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
20
global_force_attn_backend_context_manager)
21
from vllm.forward_context import set_forward_context
22
from vllm.platforms import current_platform
@@ -774,7 +774,7 @@ def set_reset_environment(attn_backend):
774
default_dtype = torch.get_default_dtype()
775
if attn_backend.name == 'FLASH_ATTN':
776
torch.set_default_dtype(torch.bfloat16)
777
- get_attn_backend.cache_clear()
777
+ _cached_get_attn_backend.cache_clear()
778
yield
779
# Reset the torch datatype to what it was before the test
780
# so as not to impact the remaining tests.
vllm/attention/selector.py
+33
-10
@@ -89,7 +89,6 @@ def get_global_forced_attn_backend() -> Optional[_Backend]:
89
return forced_attn_backend
90
91
92
-@lru_cache(maxsize=None)
92
def get_attn_backend(
93
head_size: int,
94
dtype: torch.dtype,
@@ -99,6 +98,31 @@ def get_attn_backend(
98
is_blocksparse: bool = False,
99
) -> Type[AttentionBackend]:
100
"""Selects which attention backend to use and lazily imports it."""
101
+ # Accessing envs.* behind an @lru_cache decorator can cause the wrong
102
+ # value to be returned from the cache if the value changes between calls.
103
+ # To avoid this, we read envs.VLLM_USE_V1 here and pass it explicitly to the
104
+ # private function.
105
+ return _cached_get_attn_backend(
106
+ head_size=head_size,
107
+ dtype=dtype,
108
+ kv_cache_dtype=kv_cache_dtype,
109
+ block_size=block_size,
110
+ is_attention_free=is_attention_free,
111
+ is_blocksparse=is_blocksparse,
112
+ use_v1=envs.VLLM_USE_V1,
113
+ )
114
+
115
+
116
+@lru_cache(maxsize=None)
117
+def _cached_get_attn_backend(
118
+ head_size: int,
119
+ dtype: torch.dtype,
120
+ kv_cache_dtype: Optional[str],
121
+ block_size: int,
122
+ is_attention_free: bool,
123
+ is_blocksparse: bool = False,
124
+ use_v1: bool = False,
125
+) -> Type[AttentionBackend]:
126
if is_blocksparse:
127
logger.info("Using BlocksparseFlashAttention backend.")
128
from vllm.attention.backends.blocksparse_attn import (
@@ -106,7 +130,7 @@ def get_attn_backend(
130
return BlocksparseFlashAttentionBackend
131
132
backend = which_attn_to_use(head_size, dtype, kv_cache_dtype, block_size,
109
- is_attention_free)
133
+ is_attention_free, use_v1)
134
if backend == _Backend.FLASH_ATTN:
135
logger.info("Using Flash Attention backend.")
136
from vllm.attention.backends.flash_attn import ( # noqa: F401
@@ -162,13 +186,12 @@ def get_attn_backend(
186
raise ValueError("Invalid attention backend.")
187
188
165
-def which_attn_to_use(
166
- head_size: int,
167
- dtype: torch.dtype,
168
- kv_cache_dtype: Optional[str],
169
- block_size: int,
170
- is_attention_free: bool,
171
-) -> _Backend:
189
+def which_attn_to_use(head_size: int,
190
+ dtype: torch.dtype,
191
+ kv_cache_dtype: Optional[str],
192
+ block_size: int,
193
+ is_attention_free: bool,
194
+ use_v1: bool = False) -> _Backend:
195
"""Returns which flash attention backend to use."""
196
# Default case.
197
selected_backend = _Backend.FLASH_ATTN
@@ -228,7 +251,7 @@ def which_attn_to_use(
251
if current_platform.is_hpu():
252
return _Backend.HPU_ATTN
253
231
- if envs.VLLM_USE_V1:
254
+ if use_v1:
255
return _Backend.FLASH_ATTN_VLLM_V1
256
257
# FlashAttn in NVIDIA GPUs.
vllm/engine/multiprocessing/engine.py
+10
-8
@@ -6,7 +6,9 @@ from typing import Iterator, List, Optional, Union
6
import cloudpickle
7
import zmq
8
9
+import vllm.envs
10
from vllm import AsyncEngineArgs, SamplingParams
11
+from vllm.engine.llm_engine import LLMEngine
12
# yapf conflicts with isort for this block
13
# yapf: disable
14
from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
@@ -17,17 +19,11 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
19
RPCStartupRequest, RPCStartupResponse,
20
RPCUProfileRequest)
21
# yapf: enable
20
-from vllm.envs import VLLM_USE_V1
22
from vllm.executor.gpu_executor import GPUExecutor
23
from vllm.logger import init_logger
24
from vllm.outputs import RequestOutput
25
from vllm.usage.usage_lib import UsageContext
26
26
-if VLLM_USE_V1:
27
- from vllm.v1.engine.llm_engine import LLMEngine
28
-else:
29
- from vllm.engine.llm_engine import LLMEngine
30
-
27
logger = init_logger(__name__)
28
29
POLLING_TIMEOUT_MS = 10000
@@ -117,11 +113,17 @@ class MQLLMEngine:
113
load_general_plugins()
114
115
engine_config = engine_args.create_engine_config()
116
+ if vllm.envs.VLLM_USE_V1:
117
+ # Lazy import: the v1 package isn't distributed
118
+ from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
119
+ engine_class = V1LLMEngine
120
+ else:
121
+ engine_class = LLMEngine
122
121
- executor_class = LLMEngine._get_executor_cls(engine_config)
123
+ executor_class = engine_class._get_executor_cls(engine_config)
124
125
use_async_sockets = (engine_config.model_config.use_async_output_proc
124
- and not VLLM_USE_V1)
126
+ and not vllm.envs.VLLM_USE_V1)
127
128
return cls(ipc_path=ipc_path,
129
use_async_sockets=use_async_sockets,
vllm/entrypoints/llm.py
+17
-9
@@ -1,7 +1,7 @@
1
import itertools
2
import warnings
3
from contextlib import contextmanager
4
-from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
4
+from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type,
5
Union, cast, overload)
6
7
from tqdm import tqdm
@@ -10,6 +10,7 @@ from vllm import envs
10
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
11
BeamSearchSequence, get_beam_search_score)
12
from vllm.engine.arg_utils import EngineArgs, TaskOption
13
+from vllm.engine.llm_engine import LLMEngine
14
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
15
apply_hf_chat_template,
16
apply_mistral_chat_template,
@@ -31,11 +32,6 @@ from vllm.transformers_utils.tokenizer_group import TokenizerGroup
32
from vllm.usage.usage_lib import UsageContext
33
from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
34
34
-if envs.VLLM_USE_V1:
35
- from vllm.v1.engine.llm_engine import LLMEngine # type: ignore
36
-else:
37
- from vllm.engine.llm_engine import LLMEngine # type: ignore
38
-
35
logger = init_logger(__name__)
36
37
@@ -206,10 +202,21 @@ class LLM:
202
pooling_returned_token_ids=pooling_returned_token_ids,
203
**kwargs,
204
)
209
- self.llm_engine = LLMEngine.from_engine_args(
205
+ # Logic to switch between engines is done at runtime instead of import
206
+ # to avoid import order issues
207
+ self.engine_class = self.get_engine_class()
208
+ self.llm_engine = self.engine_class.from_engine_args(
209
engine_args, usage_context=UsageContext.LLM_CLASS)
210
self.request_counter = Counter()
211
212
+ @staticmethod
213
+ def get_engine_class() -> Type[LLMEngine]:
214
+ if envs.VLLM_USE_V1:
215
+ # Lazy import: the v1 package isn't distributed
216
+ from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
217
+ return V1LLMEngine # type: ignore
218
+ return LLMEngine
219
+
220
def get_tokenizer(self) -> AnyTokenizer:
221
return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer
222
@@ -394,7 +401,7 @@ class LLM:
401
priority=priority)
402
403
outputs = self._run_engine(use_tqdm=use_tqdm)
397
- return LLMEngine.validate_outputs(outputs, RequestOutput)
404
+ return self.engine_class.validate_outputs(outputs, RequestOutput)
405
406
def beam_search(
407
self,
@@ -769,7 +776,8 @@ class LLM:
776
)
777
778
outputs = self._run_engine(use_tqdm=use_tqdm)
772
- return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
779
+ return self.engine_class.validate_outputs(outputs,
780
+ EmbeddingRequestOutput)
781
782
def start_profile(self) -> None:
783
self.llm_engine.start_profile()
vllm/model_executor/layers/sampler.py
+9
@@ -30,6 +30,15 @@ if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
30
else:
31
flashinfer_top_k_top_p_sampling = None
32
33
+
34
+def get_sampler() -> torch.nn.Module:
35
+ if envs.VLLM_USE_V1:
36
+ # Lazy import: the v1 package isn't distributed
37
+ from vllm.v1.sample.sampler import Sampler as V1Sampler
38
+ return V1Sampler()
39
+ return Sampler()
40
+
41
+
42
# (num_token_ids, num_parent_ids) per sequence group.
43
SampleResultType = List[Tuple[List[int], List[int]]]
44
vllm/model_executor/models/arctic.py
+2
-2
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.quantization.deepspeedfp import (
24
DeepSpeedFPConfig, DeepSpeedFPParameter)
25
from vllm.model_executor.layers.rotary_embedding import get_rope
26
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
27
from vllm.model_executor.layers.vocab_parallel_embedding import (
28
ParallelLMHead, VocabParallelEmbedding)
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -436,7 +436,7 @@ class ArcticForCausalLM(nn.Module, SupportsPP):
436
self.unpadded_vocab_size = config.vocab_size
437
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
438
config.vocab_size)
439
- self.sampler = Sampler()
439
+ self.sampler = get_sampler()
440
self.make_empty_intermediate_tensors = (
441
self.model.make_empty_intermediate_tensors)
442
vllm/model_executor/models/baichuan.py
+2
-2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization import QuantizationConfig
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -352,7 +352,7 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
352
if self.config.tie_word_embeddings:
353
self.lm_head.weight = self.model.embed_tokens.weight
354
self.logits_processor = LogitsProcessor(config.vocab_size)
355
- self.sampler = Sampler()
355
+ self.sampler = get_sampler()
356
self.make_empty_intermediate_tensors = (
357
self.model.make_empty_intermediate_tensors)
358
vllm/model_executor/models/bart.py
+2
-2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -838,7 +838,7 @@ class BartForConditionalGeneration(nn.Module):
838
839
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
840
config.vocab_size)
841
- self.sampler = Sampler()
841
+ self.sampler = get_sampler()
842
843
def forward(
844
self,
vllm/model_executor/models/blip2.py
+2
-2
@@ -13,7 +13,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13
InputContext, token_inputs)
14
from vllm.model_executor.layers.activation import get_act_fn
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.multimodal.utils import consecutive_placeholder_ranges
@@ -525,7 +525,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
525
if hasattr(self.language_model, "sampler"):
526
return self.language_model.sampler
527
528
- return Sampler()
528
+ return get_sampler()
529
530
def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
531
h = w = self.config.vision_config.image_size
vllm/model_executor/models/bloom.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
RowParallelLinear)
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization import QuantizationConfig
36
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -298,7 +298,7 @@ class BloomForCausalLM(nn.Module, SupportsPP):
298
self.config.hidden_size)
299
300
self.logits_processor = LogitsProcessor(config.vocab_size)
301
- self.sampler = Sampler()
301
+ self.sampler = get_sampler()
302
self.make_empty_intermediate_tensors = (
303
self.transformer.make_empty_intermediate_tensors)
304
vllm/model_executor/models/chameleon.py
+2
-2
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
21
from vllm.model_executor.layers.logits_processor import LogitsProcessor
22
from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.rotary_embedding import get_rope
24
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
26
ParallelLMHead, VocabParallelEmbedding)
27
from vllm.model_executor.model_loader.weight_utils import (
@@ -946,7 +946,7 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
946
logit_scale = getattr(config, "logit_scale", 1.0)
947
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
948
config.vocab_size, logit_scale)
949
- self.sampler = Sampler()
949
+ self.sampler = get_sampler()
950
self.make_empty_intermediate_tensors = (
951
self.model.make_empty_intermediate_tensors)
952
vllm/model_executor/models/chatglm.py
+2
-2
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
24
from vllm.model_executor.layers.logits_processor import LogitsProcessor
25
from vllm.model_executor.layers.quantization import QuantizationConfig
26
from vllm.model_executor.layers.rotary_embedding import get_rope
27
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
27
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
28
from vllm.model_executor.layers.vocab_parallel_embedding import (
29
ParallelLMHead, VocabParallelEmbedding)
30
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -616,7 +616,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
616
self.transformer.embedding.weight)
617
self.lm_head = self.transformer.output_layer
618
self.logits_processor = LogitsProcessor(config.padded_vocab_size)
619
- self.sampler = Sampler()
619
+ self.sampler = get_sampler()
620
621
def forward(self,
622
input_ids: torch.Tensor,
vllm/model_executor/models/commandr.py
+2
-2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization import QuantizationConfig
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
VocabParallelEmbedding)
43
from vllm.model_executor.model_loader.weight_utils import (
@@ -355,7 +355,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
355
cache_config,
356
quant_config,
357
lora_config=lora_config)
358
- self.sampler = Sampler()
358
+ self.sampler = get_sampler()
359
self.make_empty_intermediate_tensors = (
360
self.model.make_empty_intermediate_tensors)
361
vllm/model_executor/models/dbrx.py
+2
-2
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
14
from vllm.model_executor.layers.logits_processor import LogitsProcessor
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.layers.rotary_embedding import get_rope
17
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
20
from vllm.model_executor.model_loader.weight_utils import (
@@ -373,7 +373,7 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
373
)
374
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
375
config.vocab_size)
376
- self.sampler = Sampler()
376
+ self.sampler = get_sampler()
377
self.make_empty_intermediate_tensors = (
378
self.transformer.make_empty_intermediate_tensors)
379
vllm/model_executor/models/deepseek.py
+2
-2
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
41
from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.quantization import QuantizationConfig
43
from vllm.model_executor.layers.rotary_embedding import get_rope
44
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
44
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45
from vllm.model_executor.layers.vocab_parallel_embedding import (
46
ParallelLMHead, VocabParallelEmbedding)
47
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -399,7 +399,7 @@ class DeepseekForCausalLM(nn.Module, SupportsPP):
399
if self.config.tie_word_embeddings:
400
self.lm_head.weight = self.model.embed_tokens.weight
401
self.logits_processor = LogitsProcessor(config.vocab_size)
402
- self.sampler = Sampler()
402
+ self.sampler = get_sampler()
403
self.make_empty_intermediate_tensors = (
404
self.model.make_empty_intermediate_tensors)
405
vllm/model_executor/models/deepseek_v2.py
+2
-2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
42
from vllm.model_executor.layers.logits_processor import LogitsProcessor
43
from vllm.model_executor.layers.quantization import QuantizationConfig
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -496,7 +496,7 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
496
config.hidden_size,
497
quant_config=quant_config)
498
self.logits_processor = LogitsProcessor(config.vocab_size)
499
- self.sampler = Sampler()
499
+ self.sampler = get_sampler()
500
self.make_empty_intermediate_tensors = (
501
self.model.make_empty_intermediate_tensors)
502
vllm/model_executor/models/exaone.py
+2
-2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
get_compressed_tensors_cache_scale)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import (
@@ -478,7 +478,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
478
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
479
config.vocab_size,
480
logit_scale)
481
- self.sampler = Sampler()
481
+ self.sampler = get_sampler()
482
else:
483
self.lm_head = PPMissingLayer()
484
vllm/model_executor/models/falcon.py
+2
-2
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
38
from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
from vllm.model_executor.layers.quantization import QuantizationConfig
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -426,7 +426,7 @@ class FalconForCausalLM(nn.Module, SupportsPP):
426
quant_config=quant_config,
427
)
428
self.logits_processor = LogitsProcessor(config.vocab_size)
429
- self.sampler = Sampler()
429
+ self.sampler = get_sampler()
430
self.make_empty_intermediate_tensors = (
431
self.transformer.make_empty_intermediate_tensors)
432
vllm/model_executor/models/florence2.py
+2
-2
@@ -10,7 +10,7 @@ from vllm.config import CacheConfig
10
from vllm.model_executor.layers.logits_processor import LogitsProcessor
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig)
13
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
13
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
14
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
15
from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
16
BartParallelLMHead,
@@ -112,7 +112,7 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
112
113
self.logits_processor = LogitsProcessor(self.vocab_size,
114
config.vocab_size)
115
- self.sampler = Sampler()
115
+ self.sampler = get_sampler()
116
117
def forward(
118
self,
vllm/model_executor/models/gemma.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.quantization import QuantizationConfig
35
from vllm.model_executor.layers.rotary_embedding import get_rope
36
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -393,7 +393,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
393
quant_config,
394
prefix=maybe_prefix(prefix, "model"))
395
self.logits_processor = LogitsProcessor(config.vocab_size)
396
- self.sampler = Sampler()
396
+ self.sampler = get_sampler()
397
self.make_empty_intermediate_tensors = (
398
self.model.make_empty_intermediate_tensors)
399
vllm/model_executor/models/gemma2.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.pooler import Pooler, PoolingType
34
from vllm.model_executor.layers.quantization import QuantizationConfig
35
from vllm.model_executor.layers.rotary_embedding import get_rope
36
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -414,7 +414,7 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
414
self.model = Gemma2Model(config, cache_config, quant_config)
415
self.logits_processor = LogitsProcessor(
416
config.vocab_size, soft_cap=config.final_logit_softcapping)
417
- self.sampler = Sampler()
417
+ self.sampler = get_sampler()
418
self.make_empty_intermediate_tensors = (
419
self.model.make_empty_intermediate_tensors)
420
vllm/model_executor/models/gpt2.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
RowParallelLinear)
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization import QuantizationConfig
36
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -259,7 +259,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
259
self.lm_head = ParallelLMHead(self.config.vocab_size,
260
self.config.hidden_size)
261
self.logits_processor = LogitsProcessor(config.vocab_size)
262
- self.sampler = Sampler()
262
+ self.sampler = get_sampler()
263
self.make_empty_intermediate_tensors = (
264
self.transformer.make_empty_intermediate_tensors)
265
vllm/model_executor/models/gpt_bigcode.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
RowParallelLinear)
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization import QuantizationConfig
36
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -285,7 +285,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
285
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
286
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
287
config.vocab_size)
288
- self.sampler = Sampler()
288
+ self.sampler = get_sampler()
289
self.make_empty_intermediate_tensors = (
290
self.transformer.make_empty_intermediate_tensors)
291
vllm/model_executor/models/gpt_j.py
+2
-2
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
ParallelLMHead, VocabParallelEmbedding)
38
from vllm.model_executor.model_loader.weight_utils import (
@@ -247,7 +247,7 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
247
quant_config=quant_config,
248
)
249
self.logits_processor = LogitsProcessor(config.vocab_size)
250
- self.sampler = Sampler()
250
+ self.sampler = get_sampler()
251
self.make_empty_intermediate_tensors = (
252
self.transformer.make_empty_intermediate_tensors)
253
vllm/model_executor/models/gpt_neox.py
+2
-2
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
ParallelLMHead, VocabParallelEmbedding)
38
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -260,7 +260,7 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
260
if self.config.tie_word_embeddings:
261
self.embed_out.weight = self.gpt_neox.embed_in.weight
262
self.logits_processor = LogitsProcessor(config.vocab_size)
263
- self.sampler = Sampler()
263
+ self.sampler = get_sampler()
264
self.make_empty_intermediate_tensors = (
265
self.gpt_neox.make_empty_intermediate_tensors)
266
vllm/model_executor/models/granite.py
+2
-2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization.base_config import (
42
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
get_compressed_tensors_cache_scale)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import (
@@ -411,7 +411,7 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
411
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
412
config.vocab_size,
413
scale=logit_scale)
414
- self.sampler = Sampler()
414
+ self.sampler = get_sampler()
415
else:
416
self.lm_head = PPMissingLayer()
417
vllm/model_executor/models/granitemoe.py
+2
-2
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
from vllm.model_executor.layers.quantization.base_config import (
40
QuantizationConfig)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -371,7 +371,7 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
371
scale=1 /
372
self.config.logits_scaling)
373
374
- self.sampler = Sampler()
374
+ self.sampler = get_sampler()
375
376
def forward(
377
self,
vllm/model_executor/models/internlm2.py
+2
-2
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
20
from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.rotary_embedding import get_rope
23
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
ParallelLMHead, VocabParallelEmbedding)
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -338,7 +338,7 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
338
if self.config.tie_word_embeddings:
339
self.output.weight = self.model.tok_embeddings.weight
340
self.logits_processor = LogitsProcessor(config.vocab_size)
341
- self.sampler = Sampler()
341
+ self.sampler = get_sampler()
342
self.make_empty_intermediate_tensors = (
343
self.model.make_empty_intermediate_tensors)
344
vllm/model_executor/models/internvl.py
+2
-2
@@ -21,7 +21,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
InputContext, token_inputs)
22
from vllm.model_executor.layers.quantization import (AWQConfig,
23
QuantizationConfig)
24
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25
from vllm.model_executor.models.intern_vit import (InternVisionModel,
26
InternVisionPatchModel)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -467,7 +467,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
467
if hasattr(self.language_model, "sampler"):
468
return self.language_model.sampler
469
470
- return Sampler()
470
+ return get_sampler()
471
472
def _init_vision_model(
473
self,
vllm/model_executor/models/jais.py
+2
-2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
RowParallelLinear)
35
from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
from vllm.model_executor.layers.quantization import QuantizationConfig
37
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -308,7 +308,7 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
308
config.mup_width_scale)
309
self.logits_processor = LogitsProcessor(vocab_size=config.vocab_size,
310
scale=self.output_logits_scale)
311
- self.sampler = Sampler()
311
+ self.sampler = get_sampler()
312
self.make_empty_intermediate_tensors = (
313
self.transformer.make_empty_intermediate_tensors)
314
vllm/model_executor/models/jamba.py
+2
-2
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
17
from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
19
from vllm.model_executor.layers.quantization import QuantizationConfig
20
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
20
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21
from vllm.model_executor.layers.vocab_parallel_embedding import (
22
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -383,7 +383,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
383
384
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
385
config.vocab_size)
386
- self.sampler = Sampler()
386
+ self.sampler = get_sampler()
387
388
def forward(self,
389
input_ids: torch.Tensor,
vllm/model_executor/models/llama.py
+2
-2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
get_compressed_tensors_cache_scale)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import (
@@ -536,7 +536,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
536
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
537
config.vocab_size,
538
logit_scale)
539
- self.sampler = Sampler()
539
+ self.sampler = get_sampler()
540
else:
541
self.lm_head = PPMissingLayer()
542
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/llava.py
+2
-2
@@ -14,7 +14,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14
InputContext)
15
from vllm.model_executor.layers.activation import get_act_fn
16
from vllm.model_executor.layers.quantization import QuantizationConfig
17
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.sequence import IntermediateTensors
@@ -302,7 +302,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
302
if hasattr(self.language_model, "sampler"):
303
return self.language_model.sampler
304
305
- return Sampler()
305
+ return get_sampler()
306
307
def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
308
h = w = self.config.vision_config.image_size
vllm/model_executor/models/llava_next.py
+2
-2
@@ -16,7 +16,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16
InputContext)
17
from vllm.model_executor.layers.pooler import Pooler, PoolingType
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20
from vllm.model_executor.pooling_metadata import PoolingMetadata
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -327,7 +327,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
327
if hasattr(self.language_model, "sampler"):
328
return self.language_model.sampler
329
330
- return Sampler()
330
+ return get_sampler()
331
332
def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
333
expected_dims = (2, )
vllm/model_executor/models/llava_next_video.py
+2
-2
@@ -15,7 +15,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15
InputContext, token_inputs)
16
from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19
from vllm.model_executor.models.clip import CLIPVisionModel
20
from vllm.model_executor.sampling_metadata import SamplingMetadata
21
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -289,7 +289,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
289
if hasattr(self.language_model, "sampler"):
290
return self.language_model.sampler
291
292
- return Sampler()
292
+ return get_sampler()
293
294
def _validate_video_pixel_values(
295
self, data: Union[torch.Tensor, List[torch.Tensor]]
vllm/model_executor/models/llava_onevision.py
+2
-2
@@ -19,7 +19,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19
InputContext, token_inputs)
20
from vllm.model_executor.layers.activation import get_act_fn
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
22
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.utils import (cached_get_tokenizer,
@@ -437,7 +437,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
437
if hasattr(self.language_model, "sampler"):
438
return self.language_model.sampler
439
440
- return Sampler()
440
+ return get_sampler()
441
442
def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
443
expected_dims = (2, )
vllm/model_executor/models/mamba.py
+2
-2
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
14
from vllm.model_executor.layers.quantization.base_config import (
15
QuantizationConfig)
16
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17
from vllm.model_executor.layers.vocab_parallel_embedding import (
18
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
19
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -169,7 +169,7 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
169
170
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
171
config.vocab_size)
172
- self.sampler = Sampler()
172
+ self.sampler = get_sampler()
173
174
def forward(self,
175
input_ids: torch.Tensor,
vllm/model_executor/models/minicpm.py
+2
-2
@@ -43,7 +43,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
43
from vllm.model_executor.layers.logits_processor import LogitsProcessor
44
from vllm.model_executor.layers.quantization import QuantizationConfig
45
from vllm.model_executor.layers.rotary_embedding import get_rope
46
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
46
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
47
from vllm.model_executor.layers.vocab_parallel_embedding import (
48
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
49
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -496,7 +496,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
496
497
self.logits_processor = LogitsProcessor(unpadded_vocab_size,
498
config.vocab_size)
499
- self.sampler = Sampler()
499
+ self.sampler = get_sampler()
500
self.make_empty_intermediate_tensors = (
501
self.model.make_empty_intermediate_tensors)
502
vllm/model_executor/models/minicpmv.py
+2
-2
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
43
get_2d_sincos_pos_embed)
44
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
44
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
46
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
47
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -420,7 +420,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
420
quant_config=quant_config,
421
prefix="llm.lm_head")
422
self.logits_processor = LogitsProcessor(config.vocab_size)
423
- self.sampler = Sampler()
423
+ self.sampler = get_sampler()
424
425
self.make_empty_intermediate_tensors = (
426
self.llm.make_empty_intermediate_tensors)
vllm/model_executor/models/mixtral.py
+2
-2
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
38
from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
from vllm.model_executor.layers.quantization import QuantizationConfig
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import (
@@ -366,7 +366,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
366
self.lm_head.weight = self.model.embed_tokens.weight
367
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
368
config.vocab_size)
369
- self.sampler = Sampler()
369
+ self.sampler = get_sampler()
370
self.make_empty_intermediate_tensors = (
371
self.model.make_empty_intermediate_tensors)
372
vllm/model_executor/models/mixtral_quant.py
+2
-2
@@ -40,7 +40,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
40
from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
ParallelLMHead, VocabParallelEmbedding)
46
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -366,7 +366,7 @@ class MixtralForCausalLM(nn.Module, SupportsPP):
366
if self.config.tie_word_embeddings:
367
self.lm_head.weight = self.model.embed_tokens.weight
368
self.logits_processor = LogitsProcessor(config.vocab_size)
369
- self.sampler = Sampler()
369
+ self.sampler = get_sampler()
370
self.make_empty_intermediate_tensors = (
371
self.model.make_empty_intermediate_tensors)
372
vllm/model_executor/models/mllama.py
+2
-2
@@ -44,7 +44,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
44
RowParallelLinear)
45
from vllm.model_executor.layers.logits_processor import LogitsProcessor
46
from vllm.model_executor.layers.quantization import QuantizationConfig
47
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
48
from vllm.model_executor.layers.vocab_parallel_embedding import (
49
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -1141,7 +1141,7 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1141
)
1142
self.logits_processor = LogitsProcessor(config.output_hidden_states,
1143
config.text_config.vocab_size)
1144
- self.sampler = Sampler()
1144
+ self.sampler = get_sampler()
1145
1146
def compute_logits(
1147
self,
vllm/model_executor/models/mlp_speculator.py
+2
-2
@@ -6,7 +6,7 @@ import torch.nn as nn
6
7
from vllm.model_executor import SamplingMetadata
8
from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
9
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
10
from vllm.model_executor.layers.vocab_parallel_embedding import (
11
ParallelLMHead, VocabParallelEmbedding)
12
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -137,7 +137,7 @@ class MLPSpeculator(nn.Module):
137
self.config = config
138
self.logits_processor = LogitsProcessor(config.vocab_size,
139
config.vocab_size, 1.0)
140
- self.sampler = Sampler()
140
+ self.sampler = get_sampler()
141
142
def generate_proposals(
143
self,
vllm/model_executor/models/molmo.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.quantization import QuantizationConfig
35
from vllm.model_executor.layers.rotary_embedding import get_rope
36
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -1053,7 +1053,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1053
1054
self.logits_processor = LogitsProcessor(config.embedding_size
1055
or config.vocab_size)
1056
- self.sampler = Sampler()
1056
+ self.sampler = get_sampler()
1057
1058
self.make_empty_intermediate_tensors = (
1059
self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/mpt.py
+2
-2
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
RowParallelLinear)
17
from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
21
VocabParallelEmbedding)
22
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -281,7 +281,7 @@ class MPTForCausalLM(nn.Module, SupportsPP):
281
self.transformer = MPTModel(config, cache_config, quant_config)
282
self.lm_head = self.transformer.wte
283
self.logits_processor = LogitsProcessor(config.vocab_size)
284
- self.sampler = Sampler()
284
+ self.sampler = get_sampler()
285
self.make_empty_intermediate_tensors = (
286
self.transformer.make_empty_intermediate_tensors)
287
vllm/model_executor/models/nemotron.py
+2
-2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
36
from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
from vllm.model_executor.layers.quantization import QuantizationConfig
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
42
from vllm.model_executor.model_loader.weight_utils import (
@@ -441,7 +441,7 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
441
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
442
config.vocab_size,
443
logit_scale)
444
- self.sampler = Sampler()
444
+ self.sampler = get_sampler()
445
else:
446
self.lm_head = PPMissingLayer()
447
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/olmo.py
+2
-2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization import QuantizationConfig
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -309,7 +309,7 @@ class OlmoForCausalLM(nn.Module, SupportsPP):
309
quant_config=quant_config,
310
)
311
self.logits_processor = LogitsProcessor(config.vocab_size)
312
- self.sampler = Sampler()
312
+ self.sampler = get_sampler()
313
self.make_empty_intermediate_tensors = (
314
self.model.make_empty_intermediate_tensors)
315
vllm/model_executor/models/olmoe.py
+2
-2
@@ -28,7 +28,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
28
from vllm.model_executor.layers.logits_processor import LogitsProcessor
29
from vllm.model_executor.layers.quantization import QuantizationConfig
30
from vllm.model_executor.layers.rotary_embedding import get_rope
31
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
31
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
32
from vllm.model_executor.layers.vocab_parallel_embedding import (
33
ParallelLMHead, VocabParallelEmbedding)
34
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -323,7 +323,7 @@ class OlmoeForCausalLM(nn.Module, SupportsPP):
323
config.hidden_size,
324
quant_config=quant_config)
325
self.logits_processor = LogitsProcessor(config.vocab_size)
326
- self.sampler = Sampler()
326
+ self.sampler = get_sampler()
327
328
self.make_empty_intermediate_tensors = (
329
self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/opt.py
+2
-2
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
RowParallelLinear)
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization import QuantizationConfig
36
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -362,7 +362,7 @@ class OPTForCausalLM(nn.Module, SupportsPP):
362
self.lm_head = ParallelLMHead(config.vocab_size,
363
config.word_embed_proj_dim)
364
self.logits_processor = LogitsProcessor(config.vocab_size)
365
- self.sampler = Sampler()
365
+ self.sampler = get_sampler()
366
self.make_empty_intermediate_tensors = (
367
self.model.make_empty_intermediate_tensors)
368
vllm/model_executor/models/orion.py
+2
-2
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
20
from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.rotary_embedding import get_rope
23
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
ParallelLMHead, VocabParallelEmbedding)
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -284,7 +284,7 @@ class OrionForCausalLM(nn.Module, SupportsPP):
284
if self.config.tie_word_embeddings:
285
self.lm_head.weight = self.model.embed_tokens.weight
286
self.logits_processor = LogitsProcessor(config.vocab_size)
287
- self.sampler = Sampler()
287
+ self.sampler = get_sampler()
288
self.make_empty_intermediate_tensors = (
289
self.model.make_empty_intermediate_tensors)
290
vllm/model_executor/models/persimmon.py
+2
-2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
36
from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
from vllm.model_executor.layers.quantization import QuantizationConfig
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
ParallelLMHead, VocabParallelEmbedding)
42
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -279,7 +279,7 @@ class PersimmonForCausalLM(nn.Module, SupportsPP):
279
config.hidden_size,
280
bias=False)
281
self.logits_processor = LogitsProcessor(config.vocab_size)
282
- self.sampler = Sampler()
282
+ self.sampler = get_sampler()
283
self.make_empty_intermediate_tensors = (
284
self.model.make_empty_intermediate_tensors)
285
vllm/model_executor/models/phi.py
+2
-2
@@ -51,7 +51,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51
from vllm.model_executor.layers.logits_processor import LogitsProcessor
52
from vllm.model_executor.layers.quantization import QuantizationConfig
53
from vllm.model_executor.layers.rotary_embedding import get_rope
54
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
54
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55
from vllm.model_executor.layers.vocab_parallel_embedding import (
56
ParallelLMHead, VocabParallelEmbedding)
57
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -300,7 +300,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
300
bias=True,
301
quant_config=quant_config)
302
self.logits_processor = LogitsProcessor(config.vocab_size)
303
- self.sampler = Sampler()
303
+ self.sampler = get_sampler()
304
self.make_empty_intermediate_tensors = (
305
self.model.make_empty_intermediate_tensors)
306
vllm/model_executor/models/phi3_small.py
+2
-2
@@ -15,7 +15,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.rotary_embedding import get_rope
18
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
21
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -386,7 +386,7 @@ class Phi3SmallForCausalLM(nn.Module, SupportsPP):
386
if self.config.tie_word_embeddings:
387
self.lm_head.weight = self.model.embed_tokens.weight
388
self.logits_processor = LogitsProcessor(config.vocab_size)
389
- self.sampler = Sampler()
389
+ self.sampler = get_sampler()
390
self.make_empty_intermediate_tensors = (
391
self.model.make_empty_intermediate_tensors)
392
vllm/model_executor/models/phi3v.py
+2
-2
@@ -32,7 +32,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
32
from vllm.logger import init_logger
33
from vllm.model_executor.layers.pooler import Pooler, PoolingType
34
from vllm.model_executor.layers.quantization import QuantizationConfig
35
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
38
from vllm.model_executor.models.clip import CLIPVisionModel
@@ -570,7 +570,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
570
if hasattr(self.language_model, "sampler"):
571
return self.language_model.sampler
572
573
- return Sampler()
573
+ return get_sampler()
574
575
def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
576
expected_dims = (2, )
vllm/model_executor/models/phimoe.py
+2
-2
@@ -38,7 +38,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization.base_config import (
39
QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import (
@@ -562,7 +562,7 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
562
)
563
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
564
config.vocab_size)
565
- self.sampler = Sampler()
565
+ self.sampler = get_sampler()
566
567
self.make_empty_intermediate_tensors = (
568
self.model.make_empty_intermediate_tensors)
vllm/model_executor/models/pixtral.py
+2
-2
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
25
QKVParallelLinear,
26
RowParallelLinear)
27
from vllm.model_executor.layers.quantization import QuantizationConfig
28
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
28
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
from vllm.model_executor.models.utils import merge_multimodal_embeddings
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -190,7 +190,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
190
if hasattr(self.language_model, "sampler"):
191
return self.language_model.sampler
192
193
- return Sampler()
193
+ return get_sampler()
194
195
def forward(
196
self,
vllm/model_executor/models/qwen.py
+2
-2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
from vllm.model_executor.layers.quantization import QuantizationConfig
37
from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
ParallelLMHead, VocabParallelEmbedding)
42
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -884,7 +884,7 @@ class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
884
if self.config.tie_word_embeddings:
885
self.lm_head.weight = self.transformer.wte.weight
886
self.logits_processor = LogitsProcessor(config.vocab_size)
887
- self.sampler = Sampler()
887
+ self.sampler = get_sampler()
888
self.make_empty_intermediate_tensors = (
889
self.transformer.make_empty_intermediate_tensors)
890
vllm/model_executor/models/qwen2.py
+2
-2
@@ -39,7 +39,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
from vllm.model_executor.layers.quantization import QuantizationConfig
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import (
@@ -444,7 +444,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
444
prefix, "lm_head"))
445
446
self.logits_processor = LogitsProcessor(config.vocab_size)
447
- self.sampler = Sampler()
447
+ self.sampler = get_sampler()
448
self.make_empty_intermediate_tensors = (
449
self.model.make_empty_intermediate_tensors)
450
vllm/model_executor/models/qwen2_audio.py
+2
-2
@@ -36,7 +36,7 @@ from vllm.logger import init_logger
36
from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
from vllm.model_executor.layers.quantization.base_config import (
38
QuantizationConfig)
39
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
41
from vllm.model_executor.model_loader.weight_utils import (
42
default_weight_loader, maybe_remap_kv_scale_name)
@@ -295,7 +295,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
295
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
296
config.text_config.vocab_size,
297
logit_scale)
298
- self.sampler = Sampler()
298
+ self.sampler = get_sampler()
299
300
self.make_empty_intermediate_tensors = (
301
self.language_model.make_empty_intermediate_tensors)
vllm/model_executor/models/qwen2_moe.py
+2
-2
@@ -44,7 +44,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
44
from vllm.model_executor.layers.logits_processor import LogitsProcessor
45
from vllm.model_executor.layers.quantization import QuantizationConfig
46
from vllm.model_executor.layers.rotary_embedding import get_rope
47
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
48
from vllm.model_executor.layers.vocab_parallel_embedding import (
49
ParallelLMHead, VocabParallelEmbedding)
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -393,7 +393,7 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
393
if self.config.tie_word_embeddings:
394
self.lm_head.weight = self.model.embed_tokens.weight
395
self.logits_processor = LogitsProcessor(config.vocab_size)
396
- self.sampler = Sampler()
396
+ self.sampler = get_sampler()
397
self.make_empty_intermediate_tensors = (
398
self.model.make_empty_intermediate_tensors)
399
vllm/model_executor/models/qwen2_vl.py
+2
-2
@@ -52,7 +52,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52
RowParallelLinear)
53
from vllm.model_executor.layers.logits_processor import LogitsProcessor
54
from vllm.model_executor.layers.quantization import QuantizationConfig
55
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
55
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
56
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
57
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
58
from vllm.model_executor.models.qwen2 import Qwen2Model
@@ -990,7 +990,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
990
self.lm_head = PPMissingLayer()
991
992
self.logits_processor = LogitsProcessor(config.vocab_size)
993
- self.sampler = Sampler()
993
+ self.sampler = get_sampler()
994
self.make_empty_intermediate_tensors = (
995
make_empty_intermediate_tensors_factory(
996
["hidden_states", "residual"], config.hidden_size))
vllm/model_executor/models/solar.py
+2
-2
@@ -42,7 +42,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
42
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
get_compressed_tensors_cache_scale)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import (
@@ -449,7 +449,7 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
449
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
450
config.vocab_size,
451
logit_scale)
452
- self.sampler = Sampler()
452
+ self.sampler = get_sampler()
453
else:
454
self.lm_head = PPMissingLayer()
455
vllm/model_executor/models/stablelm.py
+2
-2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization import QuantizationConfig
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -261,7 +261,7 @@ class StablelmForCausalLM(nn.Module, SupportsPP):
261
if self.config.tie_word_embeddings:
262
self.lm_head.weight = self.model.embed_tokens.weight
263
self.logits_processor = LogitsProcessor(config.vocab_size)
264
- self.sampler = Sampler()
264
+ self.sampler = get_sampler()
265
self.make_empty_intermediate_tensors = (
266
self.model.make_empty_intermediate_tensors)
267
vllm/model_executor/models/starcoder2.py
+2
-2
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization import QuantizationConfig
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -269,7 +269,7 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
269
)
270
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
271
config.vocab_size)
272
- self.sampler = Sampler()
272
+ self.sampler = get_sampler()
273
self.make_empty_intermediate_tensors = (
274
self.model.make_empty_intermediate_tensors)
275
vllm/model_executor/models/ultravox.py
+2
-2
@@ -21,7 +21,7 @@ from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
22
from vllm.model_executor.layers.layernorm import RMSNorm
23
from vllm.model_executor.layers.quantization import QuantizationConfig
24
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25
from vllm.model_executor.model_loader.loader import DefaultModelLoader
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
@@ -379,7 +379,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
379
if hasattr(self.language_model, "sampler"):
380
return self.language_model.sampler
381
382
- return Sampler()
382
+ return get_sampler()
383
384
def _audio_features_to_embeddings(
385
self, input_features: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/xverse.py
+2
-2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization import QuantizationConfig
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -334,7 +334,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
334
if self.config.tie_word_embeddings:
335
self.lm_head.weight = self.model.embed_tokens.weight
336
self.logits_processor = LogitsProcessor(config.vocab_size)
337
- self.sampler = Sampler()
337
+ self.sampler = get_sampler()
338
self.make_empty_intermediate_tensors = (
339
self.model.make_empty_intermediate_tensors)
340
vllm/v1/attention/backends/flash_attn.py
+6
-6
@@ -136,7 +136,7 @@ class FlashAttentionImpl(AttentionImpl):
136
"key/v_scale is not supported in FlashAttention.")
137
138
output = torch.empty_like(query)
139
- torch.ops.vllm.unified_flash_attention(
139
+ torch.ops.vllm.unified_v1_flash_attention(
140
output,
141
query,
142
key,
@@ -156,7 +156,7 @@ class FlashAttentionImpl(AttentionImpl):
156
return output
157
158
159
-def unified_flash_attention(
159
+def unified_v1_flash_attention(
160
output: torch.Tensor,
161
query: torch.Tensor,
162
key: torch.Tensor,
@@ -222,7 +222,7 @@ def unified_flash_attention(
222
output[:num_actual_tokens].copy_(attn_output)
223
224
225
-def unified_flash_attention_fake(
225
+def unified_v1_flash_attention_fake(
226
output: torch.Tensor,
227
query: torch.Tensor,
228
key: torch.Tensor,
@@ -243,8 +243,8 @@ def unified_flash_attention_fake(
243
244
245
direct_register_custom_op(
246
- op_name="unified_flash_attention",
247
- op_func=unified_flash_attention,
246
+ op_name="unified_v1_flash_attention",
247
+ op_func=unified_v1_flash_attention,
248
mutates_args=["kv_cache", "output"],
249
- fake_impl=unified_flash_attention_fake,
249
+ fake_impl=unified_v1_flash_attention_fake,
250
)
vllm/v1/engine/llm_engine.py
+6
@@ -155,6 +155,12 @@ class LLMEngine:
155
# GPU and CPU blocks, which are profiled in the distributed executor.
156
self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
157
158
+ def __del__(self):
159
+ # Small hack- implicit clean up of resources on garbage collect
160
+ # TODO: this should probably be explicitly invoked when we're done with
161
+ # the engine
162
+ self.terminate_detokenizer()
163
+
164
def _initialize_kv_caches(self) -> None:
165
num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
166
)
vllm/v1/tokenizer/detokenizer.py
+4
-4
@@ -73,7 +73,7 @@ class Detokenizer:
73
return None
74
75
def terminate(self) -> None:
76
- self.push_socket.send(b"", flags=zmq.NOBLOCK)
76
+ self.detokenizer.kill()
77
self.detokenizer.join()
78
79
@@ -108,10 +108,10 @@ class DetokenizerProc(multiprocessing.Process):
108
self.push_socket.bind(f"tcp://*:{self.push_port}")
109
110
while True:
111
+ if self.pull_socket.poll(timeout=1000) == 0:
112
+ # Nothing to read
113
+ continue
114
message = self.pull_socket.recv()
112
- if message == b"":
113
- # Terminate signal.
114
- break
115
inputs = self.msgpack_decoder.decode(message)
116
117
for req_id in inputs.free_req_ids:
vllm/v1/worker/gpu_model_runner.py
+1
-4
@@ -2,7 +2,6 @@ import os
2
import time
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING, Dict, List, Optional, Set
5
-from unittest.mock import patch
5
6
import numpy as np
7
import torch
@@ -26,7 +25,6 @@ from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
25
FlashAttentionMetadata)
26
from vllm.v1.outputs import ModelRunnerOutput
27
from vllm.v1.sample.metadata import SamplingMetadata
29
-from vllm.v1.sample.sampler import Sampler
28
29
if TYPE_CHECKING:
30
from vllm.v1.core.scheduler import SchedulerOutput
@@ -418,8 +416,7 @@ class GPUModelRunner:
416
417
logger.info("Starting to load model %s...", self.model_config.model)
418
with DeviceMemoryProfiler() as m: # noqa: SIM117
421
- with patch("vllm.model_executor.layers.sampler.Sampler", Sampler):
422
- self.model = get_model(vllm_config=self.vllm_config)
419
+ self.model = get_model(vllm_config=self.vllm_config)
420
421
self.model_memory_usage = m.consumed_memory
422
logger.info("Loading model weights took %.4f GB",