A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Migrate `logits` computation and gather to `model_runner` (#3233)
Roy committed
Mar 21, 2024 at 07:25 UTC
f1c0fc391909e55fce5f109893f3c483f69a091f
35 files changed
+577
-306
.buildkite/test-pipeline.yaml
+3
@@ -49,6 +49,9 @@ steps:
49
- label: Samplers Test
50
command: pytest -v -s samplers
51
52
+- label: LogitsProcessor Test
53
+ command: pytest -v -s test_logits_processor.py
54
+
55
- label: Worker Test
56
command: pytest -v -s worker
57
tests/lora/conftest.py
+5
-2
@@ -13,6 +13,7 @@ from huggingface_hub import snapshot_download
13
import vllm
14
from vllm.config import LoRAConfig
15
from vllm.model_executor.layers.sampler import Sampler
16
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
from vllm.model_executor.model_loader import get_model
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
MergedColumnParallelLinear,
@@ -85,7 +86,8 @@ def dummy_model() -> nn.Module:
86
("outact", nn.Sigmoid()),
87
# Special handling for lm_head & sampler
88
("lm_head", ParallelLMHead(512, 10)),
88
- ("sampler", Sampler(512))
89
+ ("logits_processor", LogitsProcessor(512)),
90
+ ("sampler", Sampler())
91
]))
92
model.config = MagicMock()
93
return model
@@ -110,7 +112,8 @@ def dummy_model_gate_up() -> nn.Module:
112
("outact", nn.Sigmoid()),
113
# Special handling for lm_head & sampler
114
("lm_head", ParallelLMHead(512, 10)),
113
- ("sampler", Sampler(512))
115
+ ("logits_processor", LogitsProcessor(512)),
116
+ ("sampler", Sampler())
117
]))
118
model.config = MagicMock()
119
return model
tests/lora/test_layers.py
+36
-30
@@ -13,14 +13,14 @@ from vllm.lora.layers import (
13
QKVParallelLinearWithLora,
14
VocabParallelEmbeddingWithLoRA,
15
RowParallelLinearWithLoRA,
16
- SamplerWithLoRA,
16
+ LogitsProcessorWithLoRA,
17
LoRAMapping,
18
BaseLayerWithLoRA,
19
)
20
from vllm.lora.models import (LoRALayerWeights, convert_mapping,
21
PackedLoRALayerWeights)
22
from vllm.config import LoRAConfig
23
-from vllm.model_executor.layers.sampler import Sampler
23
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
24
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25
MergedColumnParallelLinear,
26
RowParallelLinear,
@@ -394,7 +394,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
394
@torch.inference_mode()
395
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
396
@pytest.mark.parametrize("device", CUDA_DEVICES)
397
-def test_lm_head_sampler(dist_init, num_loras, device) -> None:
397
+def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
398
399
torch.set_default_device(device)
400
max_loras = 8
@@ -402,28 +402,29 @@ def test_lm_head_sampler(dist_init, num_loras, device) -> None:
402
max_lora_rank=8,
403
lora_dtype=torch.float16)
404
405
- def create_random_sampler_layer():
405
+ def _pretest():
406
linear = ParallelLMHead(32000 + lora_config.lora_extra_vocab_size,
407
1024, 32000)
408
linear.weight.data = torch.rand_like(linear.weight.data)
409
linear.weight.data[:, 32000:] = 0
410
- sampler = Sampler(32000 + lora_config.lora_extra_vocab_size, 32000)
411
- lora_sampler = SamplerWithLoRA(sampler, 1024, linear.weight.dtype,
412
- linear.weight.device)
413
- lora_sampler.create_lora_weights(max_loras, lora_config)
410
+ logits_processor = LogitsProcessor(
411
+ 32000 + lora_config.lora_extra_vocab_size, 32000)
412
+ lora_logits_processor = LogitsProcessorWithLoRA(
413
+ logits_processor, 1024, linear.weight.dtype, linear.weight.device)
414
+ lora_logits_processor.create_lora_weights(max_loras, lora_config)
415
415
- return linear, sampler, lora_sampler
416
+ return linear, logits_processor, lora_logits_processor
417
418
for i in range(10):
419
set_random_seed(i)
420
421
id_to_index = get_random_id_to_index(num_loras, max_loras)
421
- linear, sampler, lora_sampler = create_random_sampler_layer()
422
+ linear, logits_processor, lora_logits_processor = _pretest()
423
424
# NOTE: all the generated loras share the same embeddings tensor.
425
lora_dict, _ = populate_loras(
426
id_to_index,
426
- layer=lora_sampler,
427
+ layer=lora_logits_processor,
428
layer_weights=linear.weight,
429
generate_embeddings_tensor=1024,
430
)
@@ -447,34 +448,37 @@ def test_lm_head_sampler(dist_init, num_loras, device) -> None:
448
32000,
449
lora_config.lora_extra_vocab_size,
450
)
450
- lora_sampler.set_mapping(*mapping_info, )
451
+ lora_logits_processor.set_mapping(*mapping_info, )
452
452
- lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
453
- embedding=linear.weight,
454
- embedding_bias=None)
453
+ lora_result = lora_logits_processor._get_logits(
454
+ hidden_states=torch.cat(inputs),
455
+ embedding=linear.weight,
456
+ embedding_bias=None)
457
458
original_weight = linear.weight.clone()
459
458
- linear.weight[sampler.org_vocab_size:sampler.org_vocab_size +
460
+ linear.weight[logits_processor.
461
+ org_vocab_size:logits_processor.org_vocab_size +
462
embeddings_tensor_len] = embeddings_tensor
463
461
- sampler.org_vocab_size = 32000 + lora_config.lora_extra_vocab_size
464
+ logits_processor.org_vocab_size = (32000 +
465
+ lora_config.lora_extra_vocab_size)
466
expected_results = []
467
for input_, lora_id in zip(inputs, prompt_mapping):
468
lora = lora_dict[lora_id]
465
- result = sampler._get_logits(hidden_states=input_,
466
- embedding=linear.weight,
467
- embedding_bias=None)
469
+ result = logits_processor._get_logits(hidden_states=input_,
470
+ embedding=linear.weight,
471
+ embedding_bias=None)
472
result[:, 32000 + embeddings_tensor_len:] = float("-inf")
473
result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
474
expected_results.append(result)
475
expected_result = torch.cat(expected_results)
472
- sampler.org_vocab_size = 32000
476
+ logits_processor.org_vocab_size = 32000
477
478
# Check that resetting the lora weights succeeds
479
480
for slot_idx in range(max_loras):
477
- lora_sampler.reset_lora(slot_idx)
481
+ lora_logits_processor.reset_lora(slot_idx)
482
483
inputs, index_mapping, prompt_mapping = create_random_inputs(
484
active_lora_ids=[0],
@@ -488,14 +492,16 @@ def test_lm_head_sampler(dist_init, num_loras, device) -> None:
492
mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
493
32000,
494
lora_config.lora_extra_vocab_size)
491
- lora_sampler.set_mapping(*mapping_info, )
492
-
493
- lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
494
- embedding=original_weight,
495
- embedding_bias=None)[:, :32000]
496
- expected_result = sampler._get_logits(hidden_states=torch.cat(inputs),
497
- embedding=original_weight,
498
- embedding_bias=None)
495
+ lora_logits_processor.set_mapping(*mapping_info, )
496
+
497
+ lora_result = lora_logits_processor._get_logits(
498
+ hidden_states=torch.cat(inputs),
499
+ embedding=original_weight,
500
+ embedding_bias=None)[:, :32000]
501
+ expected_result = logits_processor._get_logits(
502
+ hidden_states=torch.cat(inputs),
503
+ embedding=original_weight,
504
+ embedding_bias=None)
505
506
rtol, atol = TOLERANCES[lora_result.dtype]
507
assert torch.allclose(lora_result,
tests/samplers/test_sampler.py
+20
-75
@@ -15,17 +15,12 @@ from vllm.worker.model_runner import ModelRunner
15
16
class MockLogitsSampler(Sampler):
17
18
- def __init__(self, vocab_size: int, fake_logits: torch.Tensor):
19
- super().__init__(vocab_size=vocab_size)
18
+ def __init__(self, fake_logits: torch.Tensor):
19
+ super().__init__()
20
self.fake_logits = fake_logits
21
22
def forward(self, *args, **kwargs):
23
- with patch(
24
- "vllm.model_executor.layers.sampler._prune_hidden_states",
25
- lambda x, y: x), patch(
26
- "vllm.model_executor.layers.sampler.Sampler._get_logits",
27
- lambda *args, **kwargs: self.fake_logits):
28
- return super().forward(*args, **kwargs)
23
+ return super().forward(*args, **kwargs)
24
25
26
def _prepare_test(
@@ -36,7 +31,7 @@ def _prepare_test(
31
fake_logits = torch.full((batch_size, vocab_size),
32
1e-2,
33
dtype=input_tensor.dtype)
39
- sampler = MockLogitsSampler(32000, fake_logits)
34
+ sampler = MockLogitsSampler(fake_logits)
35
model_runner = ModelRunner(None, None, None, None, None)
36
return input_tensor, fake_logits, sampler, model_runner
37
@@ -70,9 +65,7 @@ def _do_sample(
65
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
66
prompt_lens,
67
subquery_lens=prompt_lens)
73
- return sampler(embedding=None,
74
- hidden_states=input_tensor,
75
- sampling_metadata=sampling_metadata)
68
+ return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
69
70
71
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -85,8 +78,8 @@ def test_sampler_all_greedy(seed: int, device: str):
78
batch_size)
79
80
sampling_params = SamplingParams(temperature=0)
88
- sampler_output = _do_sample(batch_size, input_tensor, sampler,
89
- model_runner, sampling_params)
81
+ sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
82
+ sampling_params)
83
expected = torch.argmax(fake_logits, dim=-1)
84
for i, sequence_output in enumerate(sampler_output):
85
for nth_output in sequence_output.samples:
@@ -111,8 +104,8 @@ def test_sampler_all_random(seed: int, device: str):
104
temperature=1.0,
105
n=random.randint(1, 10),
106
)
114
- sampler_output = _do_sample(batch_size, input_tensor, sampler,
115
- model_runner, sampling_params)
107
+ sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
108
+ sampling_params)
109
110
for i, sequence_output in enumerate(sampler_output):
111
for nth_output in sequence_output.samples:
@@ -127,8 +120,7 @@ def test_sampler_all_random_seed(seed: int, device: str):
120
set_random_seed(seed)
121
torch.set_default_device(device)
122
batch_size = random.randint(1, 256)
130
- input_tensor, fake_logits, sampler, model_runner = _prepare_test(
131
- batch_size)
123
+ _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
124
125
for i in range(batch_size):
126
fake_logits[i, i] = 1e2
@@ -138,8 +130,8 @@ def test_sampler_all_random_seed(seed: int, device: str):
130
n=random.randint(1, 10),
131
seed=random.randint(0, 10000),
132
)
141
- sampler_output = _do_sample(batch_size, input_tensor, sampler,
142
- model_runner, sampling_params)
133
+ sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
134
+ sampling_params)
135
136
for i, sequence_output in enumerate(sampler_output):
137
for nth_output in sequence_output.samples:
@@ -154,18 +146,17 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
146
set_random_seed(seed)
147
torch.set_default_device(device)
148
batch_size = random.randint(1, 256)
157
- input_tensor, fake_logits, sampler, model_runner = _prepare_test(
158
- batch_size)
149
+ _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
150
151
sampling_params = SamplingParams(
152
temperature=1.0,
153
n=random.randint(1, 10),
154
seed=random.randint(0, 10000),
155
)
165
- first_sampler_output = _do_sample(batch_size, input_tensor, sampler,
156
+ first_sampler_output = _do_sample(batch_size, fake_logits, sampler,
157
model_runner, sampling_params)
158
168
- second_sampler_output = _do_sample(batch_size, input_tensor, sampler,
159
+ second_sampler_output = _do_sample(batch_size, fake_logits, sampler,
160
model_runner, sampling_params)
161
162
assert first_sampler_output == second_sampler_output
@@ -179,15 +170,14 @@ def test_sampler_all_beam(seed: int, device: str):
170
set_random_seed(seed)
171
torch.set_default_device(device)
172
batch_size = random.randint(1, 256)
182
- input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
173
+ _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
174
175
sampling_params = SamplingParams(
176
temperature=0,
177
best_of=2,
178
use_beam_search=True,
179
)
189
- _do_sample(batch_size, input_tensor, sampler, model_runner,
190
- sampling_params)
180
+ _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params)
181
# no assertion here as I am not sure how to determine whether
182
# the outputs are expected - in other words, this just tests
183
# whether there are no exceptions in the sampler
@@ -246,8 +236,7 @@ def test_sampler_mixed(seed: int, device: str):
236
def test_sampling(model_runner: ModelRunner):
237
sampling_metadata = model_runner._prepare_sample(
238
seq_group_metadata_list, prompt_lens, subquery_lens=prompt_lens)
249
- sampler_output = sampler(embedding=None,
250
- hidden_states=input_tensor,
239
+ sampler_output = sampler(logits=fake_logits,
240
sampling_metadata=sampling_metadata)
241
242
for i, (sequence_output, metadata) in enumerate(
@@ -294,48 +283,6 @@ def test_sampler_mixed(seed: int, device: str):
283
del model_runner
284
285
297
-@pytest.mark.parametrize("seed", RANDOM_SEEDS)
298
-@pytest.mark.parametrize("device", CUDA_DEVICES)
299
-def test_sampler_logits_processors(seed: int, device: str):
300
- set_random_seed(seed)
301
- torch.set_default_device(device)
302
- batch_size = random.randint(1, 256)
303
- input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
304
-
305
- # This sample logits processor gives maximum score to the i-th token,
306
- # where i is the length of the input sequence.
307
- # We therefore expect the output token sequence to be [0, 1, 2, ...]
308
- def pick_ith(token_ids, logits):
309
- logits[len(token_ids)] = torch.finfo(logits.dtype).max
310
- return logits
311
-
312
- seq_group_metadata_list = []
313
- prompt_lens = []
314
- for i in range(batch_size):
315
- seq_group_metadata_list.append(
316
- SequenceGroupMetadata(
317
- request_id=f"test_{i}",
318
- is_prompt=True,
319
- seq_data={0: SequenceData([1, 2, 3])},
320
- sampling_params=SamplingParams(temperature=0,
321
- logits_processors=[pick_ith]),
322
- block_tables={0: [1]},
323
- ))
324
- prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
325
-
326
- sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
327
- prompt_lens,
328
- subquery_lens=prompt_lens)
329
- sampler_output = sampler(embedding=None,
330
- hidden_states=input_tensor,
331
- sampling_metadata=sampling_metadata)
332
- for _, sequence_output in enumerate(sampler_output):
333
- for idx, nth_output in enumerate(sequence_output.samples):
334
- assert nth_output.output_token == idx
335
-
336
- del model_runner
337
-
338
-
286
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
287
@pytest.mark.parametrize("device", CUDA_DEVICES)
288
def test_sampler_top_k_top_p(seed: int, device: str):
@@ -352,7 +299,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
299
size=(batch_size, vocab_size),
300
device=input_tensor.device,
301
dtype=input_tensor.dtype)
355
- sampler = MockLogitsSampler(32000, fake_logits)
302
+ sampler = MockLogitsSampler(fake_logits)
303
model_runner = ModelRunner(None, None, None, None, None)
304
305
generation_model = GenerationMixin()
@@ -391,9 +338,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
338
return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
339
340
with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
394
- sampler(embedding=None,
395
- hidden_states=input_tensor,
396
- sampling_metadata=sampling_metadata)
341
+ sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
342
hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
343
hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
344
assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
tests/test_logits_processor.py
new
+94
@@ -0,0 +1,94 @@
1
+import random
2
+from typing import Tuple
3
+from unittest.mock import patch
4
+
5
+import pytest
6
+import torch
7
+
8
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
+from vllm.model_executor.utils import set_random_seed
10
+from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11
+from vllm.worker.model_runner import ModelRunner
12
+
13
+
14
+class MockLogitsProcessor(LogitsProcessor):
15
+
16
+ def __init__(self, vocab_size: int, scale: float,
17
+ fake_logits: torch.Tensor):
18
+ super().__init__(vocab_size=vocab_size, scale=scale)
19
+ self.fake_logits = fake_logits.clone()
20
+
21
+ def forward(self, *args, **kwargs):
22
+ with patch(
23
+ "vllm.model_executor.layers.logits_processor._prune_hidden_states",
24
+ lambda x, y: x
25
+ ), patch(
26
+ "vllm.model_executor.layers.logits_processor.LogitsProcessor._get_logits",
27
+ lambda *args, **kwargs: self.fake_logits):
28
+ return super().forward(*args, **kwargs)
29
+
30
+
31
+def _prepare_test(
32
+ batch_size: int
33
+) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor, ModelRunner]:
34
+ vocab_size = 32000
35
+ input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
36
+ fake_logits = torch.full((batch_size, vocab_size),
37
+ 1e-2,
38
+ dtype=input_tensor.dtype)
39
+ logits_processor = MockLogitsProcessor(32000, 0.5, fake_logits)
40
+ model_runner = ModelRunner(None, None, None, None, None)
41
+ return input_tensor, fake_logits, logits_processor, model_runner
42
+
43
+
44
+RANDOM_SEEDS = list(range(128))
45
+CUDA_DEVICES = [
46
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
47
+]
48
+
49
+
50
+@pytest.mark.parametrize("seed", RANDOM_SEEDS)
51
+@pytest.mark.parametrize("device", CUDA_DEVICES)
52
+def test_logits_processors(seed: int, device: str):
53
+ set_random_seed(seed)
54
+ torch.set_default_device(device)
55
+ batch_size = random.randint(1, 256)
56
+ input_tensor, fake_logits, logits_processor, model_runner = _prepare_test(
57
+ batch_size)
58
+
59
+ # This sample logits processor gives infinite score to the i-th token,
60
+ # where i is the length of the input sequence.
61
+ # We therefore expect the output token sequence to be [0, 1, 2, ...]
62
+ def pick_ith(token_ids, logits):
63
+ logits[len(token_ids)] = float("inf")
64
+ return logits
65
+
66
+ seq_group_metadata_list = []
67
+ prompt_lens = []
68
+ for i in range(batch_size):
69
+ seq_group_metadata_list.append(
70
+ SequenceGroupMetadata(
71
+ request_id=f"test_{i}",
72
+ is_prompt=True,
73
+ seq_data={0: SequenceData([1, 2, 3])},
74
+ sampling_params=SamplingParams(temperature=0,
75
+ logits_processors=[pick_ith]),
76
+ block_tables={0: [1]},
77
+ ))
78
+ prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
79
+
80
+ sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
81
+ prompt_lens,
82
+ subquery_lens=prompt_lens)
83
+ logits_processor_output = logits_processor(
84
+ embedding=None,
85
+ hidden_states=input_tensor,
86
+ sampling_metadata=sampling_metadata)
87
+
88
+ assert torch.isinf(logits_processor_output[:, 0]).all()
89
+
90
+ fake_logits *= logits_processor.scale
91
+ assert torch.allclose(logits_processor_output[:, 1], fake_logits[:, 1],
92
+ 1e-4)
93
+
94
+ del model_runner
vllm/lora/layers.py
+12
-8
@@ -10,7 +10,6 @@ from transformers import PretrainedConfig
10
11
from vllm.config import LoRAConfig
12
from vllm.lora.punica import add_lora, add_lora_slice, bgmv
13
-from vllm.model_executor.layers.sampler import Sampler
13
from vllm.model_executor.parallel_utils.communication_op import (
14
tensor_model_parallel_all_gather,
15
tensor_model_parallel_all_reduce,
@@ -20,6 +19,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
RowParallelLinear,
20
QKVParallelLinear,
21
MergedColumnParallelLinear)
22
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
VocabParallelEmbedding, ParallelLMHead)
25
from vllm.model_executor.parallel_utils.parallel_state import (
@@ -783,11 +783,11 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
783
return self.base_layer.weight
784
785
786
-class SamplerWithLoRA(BaseLayerWithLoRA):
786
+class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
787
788
def __init__(
789
self,
790
- base_layer: Sampler,
790
+ base_layer: LogitsProcessor,
791
hidden_size: int,
792
dtype: torch.dtype,
793
device: torch.device,
@@ -806,6 +806,10 @@ class SamplerWithLoRA(BaseLayerWithLoRA):
806
def vocab_size(self):
807
return self.base_layer.vocab_size
808
809
+ @property
810
+ def scale(self):
811
+ return self.base_layer.scale
812
+
813
@property
814
def org_vocab_size(self):
815
return self.base_layer.org_vocab_size
@@ -968,14 +972,14 @@ def from_layer(
972
return layer
973
974
971
-def from_layer_sampler(
972
- layer: Sampler,
975
+def from_layer_logits_processor(
976
+ layer: LogitsProcessor,
977
lm_head: ParallelLMHead,
978
max_loras: int,
979
lora_config: LoRAConfig,
980
model_config: Optional[PretrainedConfig] = None,
977
-) -> SamplerWithLoRA:
978
- ret = SamplerWithLoRA(layer, lm_head.embedding_dim, lm_head.weight.dtype,
979
- lm_head.weight.device)
981
+) -> LogitsProcessorWithLoRA:
982
+ ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
983
+ lm_head.weight.dtype, lm_head.weight.device)
984
ret.create_lora_weights(max_loras, lora_config, model_config)
985
return ret
vllm/lora/models.py
+8
-5
@@ -14,7 +14,7 @@ from vllm.config import LoRAConfig
14
from vllm.utils import LRUCache, in_wsl
15
16
from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
17
- from_layer_sampler)
17
+ from_layer_logits_processor)
18
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19
from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
20
@@ -421,11 +421,14 @@ class LoRAModelManager:
421
self.model.config))
422
# (yard1): TODO make this more robust
423
if "lm_head" in module_name:
424
- sampler_module = self.model.get_submodule("sampler")
424
+ logits_processor_module = self.model.get_submodule(
425
+ "logits_processor")
426
new_module = replace_submodule(
426
- self.model, "sampler",
427
- from_layer_sampler(sampler_module, module, self.lora_slots,
428
- self.lora_config, self.model.config))
427
+ self.model, "logits_processor",
428
+ from_layer_logits_processor(logits_processor_module,
429
+ module, self.lora_slots,
430
+ self.lora_config,
431
+ self.model.config))
432
self.register_module(module_name, new_module)
433
self._register_packed_modules(module_name)
434
new_module.set_mapping(self.base_indices, self.sampler_indices,
vllm/model_executor/layers/logits_processor.py
new
+106
@@ -0,0 +1,106 @@
1
+"""A layer that compute logits from hidden_stats."""
2
+from typing import Optional
3
+
4
+import torch
5
+import torch.nn as nn
6
+
7
+from vllm.utils import is_neuron
8
+
9
+from vllm.model_executor.parallel_utils.communication_op import (
10
+ tensor_model_parallel_gather)
11
+from vllm.model_executor.sampling_metadata import SamplingMetadata
12
+
13
+
14
+class LogitsProcessor(nn.Module):
15
+ """Process logits and apply logits processors from sampling metadata.
16
+
17
+ This layer does the following:
18
+ 1. Gather logits from model hidden_states.
19
+ 2. Scale logits if needed.
20
+ 3. Apply logits processors (if any).
21
+ """
22
+
23
+ def __init__(self,
24
+ vocab_size: int,
25
+ org_vocab_size: Optional[int] = None,
26
+ scale: Optional[float] = 1.0) -> None:
27
+ """
28
+ Args:
29
+ scale: A scaling factor to apply to the logits.
30
+ """
31
+ super().__init__()
32
+ self.scale = scale
33
+ self.vocab_size = vocab_size
34
+ # Transformers-neuronx generate outputs as logits directly.
35
+ self.logits_as_hidden_states = is_neuron()
36
+ # original vocabulary size (without LoRA).
37
+ self.org_vocab_size = org_vocab_size or vocab_size
38
+
39
+ def forward(
40
+ self,
41
+ embedding: torch.Tensor,
42
+ hidden_states: torch.Tensor,
43
+ sampling_metadata: SamplingMetadata,
44
+ embedding_bias: Optional[torch.Tensor] = None,
45
+ ) -> torch.Tensor:
46
+ if self.logits_as_hidden_states:
47
+ logits = hidden_states
48
+ else:
49
+ hidden_states = _prune_hidden_states(hidden_states,
50
+ sampling_metadata)
51
+
52
+ # Get the logits for the next tokens.
53
+ logits = self._get_logits(hidden_states, embedding, embedding_bias)
54
+
55
+ if logits is not None:
56
+ logits *= self.scale
57
+
58
+ # Apply logits processors (if any).
59
+ logits = _apply_logits_processors(logits, sampling_metadata)
60
+
61
+ return logits
62
+
63
+ def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
64
+ embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
65
+ # Get the logits for the next tokens.
66
+ logits = torch.matmul(hidden_states, embedding.t())
67
+ if embedding_bias is not None:
68
+ logits += embedding_bias
69
+ logits = tensor_model_parallel_gather(logits)
70
+ # Remove paddings in vocab (if any).
71
+ if logits is not None:
72
+ logits = logits[:, :self.org_vocab_size]
73
+ return logits
74
+
75
+
76
+def _prune_hidden_states(
77
+ hidden_states: torch.Tensor,
78
+ sampling_metadata: SamplingMetadata,
79
+) -> torch.Tensor:
80
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
81
+ return hidden_states.index_select(0,
82
+ sampling_metadata.selected_token_indices)
83
+
84
+
85
+def _apply_logits_processors(
86
+ logits: torch.Tensor,
87
+ sampling_metadata: SamplingMetadata,
88
+) -> torch.Tensor:
89
+ logits_row_idx = 0
90
+ found_logits_processors = False
91
+ for seq_ids, sampling_params in sampling_metadata.seq_groups:
92
+ logits_processors = sampling_params.logits_processors
93
+ if logits_processors:
94
+ found_logits_processors = True
95
+ for seq_id in seq_ids:
96
+ logits_row = logits[logits_row_idx]
97
+ token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
98
+ for logits_processor in logits_processors:
99
+ logits_row = logits_processor(token_ids, logits_row)
100
+ logits[logits_row_idx] = logits_row
101
+ logits_row_idx += 1
102
+ else:
103
+ logits_row_idx += len(seq_ids)
104
+ if found_logits_processors:
105
+ assert logits_row_idx == logits.shape[0]
106
+ return logits
vllm/model_executor/layers/sampler.py
+1
-80
@@ -4,8 +4,6 @@ from typing import Dict, List, Optional, Tuple
4
import torch
5
import torch.nn as nn
6
7
-from vllm.model_executor.parallel_utils.communication_op import (
8
- tensor_model_parallel_gather)
7
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
8
SamplingTensors)
9
from vllm.sampling_params import SamplingParams, SamplingType
@@ -13,7 +11,6 @@ from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
11
SamplerOutput, SequenceData, SequenceGroupOutput,
12
SequenceOutput)
13
from vllm.model_executor.layers.ops.sample import (sample as sample_triton)
16
-from vllm.utils import is_neuron
14
15
16
class Sampler(nn.Module):
@@ -31,58 +28,14 @@ class Sampler(nn.Module):
28
parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
29
"""
30
34
- def __init__(self,
35
- vocab_size: int,
36
- org_vocab_size: Optional[int] = None) -> None:
37
- super().__init__()
38
- self.vocab_size = vocab_size
39
- # Transformers-neuronx generate outputs as logits directly.
40
- self.logits_as_hidden_states = is_neuron()
41
- # original vocabulary size (without LoRA).
42
- self.org_vocab_size = org_vocab_size or vocab_size
43
-
44
- def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
45
- embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
46
- # Get the logits for the next tokens.
47
- logits = torch.matmul(hidden_states, embedding.t())
48
- if embedding_bias is not None:
49
- logits += embedding_bias
50
- logits = tensor_model_parallel_gather(logits)
51
- # Remove paddings in vocab (if any).
52
- if logits is not None:
53
- logits = logits[:, :self.org_vocab_size]
54
- return logits
55
-
31
def forward(
32
self,
58
- embedding: torch.Tensor,
59
- hidden_states: torch.Tensor,
33
+ logits: torch.Tensor,
34
sampling_metadata: SamplingMetadata,
61
- embedding_bias: Optional[torch.Tensor] = None,
35
) -> Optional[SamplerOutput]:
63
- # Get the hidden states that we use for sampling.
64
- if self.logits_as_hidden_states:
65
- logits = hidden_states
66
- else:
67
- hidden_states = _prune_hidden_states(hidden_states,
68
- sampling_metadata)
69
-
70
- # Get the logits for the next tokens.
71
- logits = self._get_logits(hidden_states, embedding, embedding_bias)
72
-
73
- # Only perform sampling in the driver worker.
74
- # Note: `_get_logits` is still distributed across TP workers because
75
- # the `embedding` weight is distributed across TP workers.
76
- # TODO(zhuohan): Change the get_logits part to a separate stage.
77
- if not sampling_metadata.perform_sampling:
78
- return None
79
-
36
assert logits is not None
37
_, vocab_size = logits.shape
38
83
- # Apply logits processors (if any).
84
- logits = _apply_logits_processors(logits, sampling_metadata)
85
-
39
# Prepare sampling tensors with pinned memory to avoid blocking.
40
(sampling_tensors, do_penalties, do_top_p_top_k,
41
do_min_p) = SamplingTensors.from_sampling_metadata(
@@ -124,14 +77,6 @@ class Sampler(nn.Module):
77
prompt_logprobs, sample_logprobs)
78
79
127
-def _prune_hidden_states(
128
- hidden_states: torch.Tensor,
129
- sampling_metadata: SamplingMetadata,
130
-) -> torch.Tensor:
131
- return hidden_states.index_select(0,
132
- sampling_metadata.selected_token_indices)
133
-
134
-
80
def _get_bin_counts_and_mask(
81
tokens: torch.Tensor,
82
vocab_size: int,
@@ -149,30 +94,6 @@ def _get_bin_counts_and_mask(
94
return bin_counts, mask
95
96
152
-def _apply_logits_processors(
153
- logits: torch.Tensor,
154
- sampling_metadata: SamplingMetadata,
155
-) -> torch.Tensor:
156
- logits_row_idx = 0
157
- found_logits_processors = False
158
- for seq_ids, sampling_params in sampling_metadata.seq_groups:
159
- logits_processors = sampling_params.logits_processors
160
- if logits_processors:
161
- found_logits_processors = True
162
- for seq_id in seq_ids:
163
- logits_row = logits[logits_row_idx]
164
- token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
165
- for logits_processor in logits_processors:
166
- logits_row = logits_processor(token_ids, logits_row)
167
- logits[logits_row_idx] = logits_row
168
- logits_row_idx += 1
169
- else:
170
- logits_row_idx += len(seq_ids)
171
- if found_logits_processors:
172
- assert logits_row_idx == logits.shape[0]
173
- return logits
174
-
175
-
97
def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
98
output_tokens_tensor: torch.Tensor,
99
presence_penalties: torch.Tensor,
vllm/model_executor/models/baichuan.py
+11
-4
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
34
QKVParallelLinear,
35
RowParallelLinear)
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.sampler import Sampler
39
from vllm.model_executor.layers.vocab_parallel_embedding import (
40
VocabParallelEmbedding, ParallelLMHead)
@@ -295,7 +296,8 @@ class BaiChuanBaseForCausalLM(nn.Module):
296
self.linear_method = linear_method
297
self.model = BaiChuanModel(config, position_embedding, linear_method)
298
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
298
- self.sampler = Sampler(config.vocab_size)
299
+ self.logits_processor = LogitsProcessor(config.vocab_size)
300
+ self.sampler = Sampler()
301
302
def forward(
303
self,
@@ -308,13 +310,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
310
input_metadata)
311
return hidden_states
312
313
+ def compute_logits(self, hidden_states: torch.Tensor,
314
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
315
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
316
+ sampling_metadata)
317
+ return logits
318
+
319
def sample(
320
self,
313
- hidden_states: torch.Tensor,
321
+ logits: torch.Tensor,
322
sampling_metadata: SamplingMetadata,
323
) -> Optional[SamplerOutput]:
316
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
317
- sampling_metadata)
324
+ next_tokens = self.sampler(logits, sampling_metadata)
325
return next_tokens
326
327
def load_weights(self,
vllm/model_executor/models/bloom.py
+11
-4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
32
RowParallelLinear)
33
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding)
@@ -273,7 +274,8 @@ class BloomForCausalLM(nn.Module):
274
self.linear_method = linear_method
275
self.transformer = BloomModel(config, linear_method)
276
self.lm_head_weight = self.transformer.word_embeddings.weight
276
- self.sampler = Sampler(config.vocab_size)
277
+ self.logits_processor = LogitsProcessor(config.vocab_size)
278
+ self.sampler = Sampler()
279
280
def forward(
281
self,
@@ -286,13 +288,18 @@ class BloomForCausalLM(nn.Module):
288
input_metadata)
289
return hidden_states
290
291
+ def compute_logits(self, hidden_states: torch.Tensor,
292
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
293
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
294
+ sampling_metadata)
295
+ return logits
296
+
297
def sample(
298
self,
291
- hidden_states: torch.Tensor,
299
+ logits: torch.Tensor,
300
sampling_metadata: SamplingMetadata,
301
) -> Optional[SamplerOutput]:
294
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
295
- sampling_metadata)
302
+ next_tokens = self.sampler(logits, sampling_metadata)
303
return next_tokens
304
305
def load_weights(self,
vllm/model_executor/models/chatglm.py
+11
-4
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
17
QKVParallelLinear,
18
RowParallelLinear)
19
from vllm.model_executor.layers.rotary_embedding import get_rope
20
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
from vllm.model_executor.layers.sampler import Sampler
22
from vllm.model_executor.layers.vocab_parallel_embedding import (
23
VocabParallelEmbedding, ParallelLMHead)
@@ -332,7 +333,8 @@ class ChatGLMForCausalLM(nn.Module):
333
self.linear_method = linear_method
334
self.transformer = ChatGLMModel(config, linear_method)
335
self.lm_head_weight = self.transformer.output_layer.weight
335
- self.sampler = Sampler(config.padded_vocab_size)
336
+ self.logits_processor = LogitsProcessor(config.padded_vocab_size)
337
+ self.sampler = Sampler()
338
339
def forward(
340
self,
@@ -345,13 +347,18 @@ class ChatGLMForCausalLM(nn.Module):
347
input_metadata)
348
return hidden_states
349
350
+ def compute_logits(self, hidden_states: torch.Tensor,
351
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
352
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
353
+ sampling_metadata)
354
+ return logits
355
+
356
def sample(
357
self,
350
- hidden_states: torch.Tensor,
358
+ logits: torch.Tensor,
359
sampling_metadata: SamplingMetadata,
360
) -> Optional[SamplerOutput]:
353
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
354
- sampling_metadata)
361
+ next_tokens = self.sampler(logits, sampling_metadata)
362
return next_tokens
363
364
def load_weights(self,
vllm/model_executor/models/deepseek.py
+11
-4
@@ -38,6 +38,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
38
QKVParallelLinear,
39
RowParallelLinear)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
VocabParallelEmbedding, ParallelLMHead)
@@ -372,7 +373,8 @@ class DeepseekForCausalLM(nn.Module):
373
self.linear_method = linear_method
374
self.model = DeepseekModel(config, linear_method)
375
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
375
- self.sampler = Sampler(config.vocab_size)
376
+ self.logits_processor = LogitsProcessor(config.vocab_size)
377
+ self.sampler = Sampler()
378
379
def forward(
380
self,
@@ -385,13 +387,18 @@ class DeepseekForCausalLM(nn.Module):
387
input_metadata)
388
return hidden_states
389
390
+ def compute_logits(self, hidden_states: torch.Tensor,
391
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
392
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
393
+ sampling_metadata)
394
+ return logits
395
+
396
def sample(
397
self,
390
- hidden_states: Optional[torch.Tensor],
398
+ logits: Optional[torch.Tensor],
399
sampling_metadata: SamplingMetadata,
400
) -> Optional[SamplerOutput]:
393
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
394
- sampling_metadata)
401
+ next_tokens = self.sampler(logits, sampling_metadata)
402
return next_tokens
403
404
def load_weights(self,
vllm/model_executor/models/falcon.py
+11
-4
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
QKVParallelLinear,
35
RowParallelLinear)
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.sampler import Sampler
39
from vllm.model_executor.layers.vocab_parallel_embedding import (
40
VocabParallelEmbedding, ParallelLMHead)
@@ -373,7 +374,8 @@ class FalconForCausalLM(nn.Module):
374
config.vocab_size,
375
config.hidden_size,
376
)
376
- self.sampler = Sampler(config.vocab_size)
377
+ self.logits_processor = LogitsProcessor(config.vocab_size)
378
+ self.sampler = Sampler()
379
380
def forward(
381
self,
@@ -390,13 +392,18 @@ class FalconForCausalLM(nn.Module):
392
)
393
return hidden_states
394
395
+ def compute_logits(self, hidden_states: torch.Tensor,
396
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
397
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
398
+ sampling_metadata)
399
+ return logits
400
+
401
def sample(
402
self,
395
- hidden_states: torch.Tensor,
403
+ logits: torch.Tensor,
404
sampling_metadata: SamplingMetadata,
405
) -> Optional[SamplerOutput]:
398
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
399
- sampling_metadata)
406
+ next_tokens = self.sampler(logits, sampling_metadata)
407
return next_tokens
408
409
def load_weights(self,
vllm/model_executor/models/gemma.py
+11
-4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
30
QKVParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.rotary_embedding import get_rope
33
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding)
@@ -281,7 +282,8 @@ class GemmaForCausalLM(nn.Module):
282
self.config = config
283
self.linear_method = linear_method
284
self.model = GemmaModel(config, linear_method)
284
- self.sampler = Sampler(config.vocab_size)
285
+ self.logits_processor = LogitsProcessor(config.vocab_size)
286
+ self.sampler = Sampler()
287
288
@torch.no_grad()
289
def forward(
@@ -295,13 +297,18 @@ class GemmaForCausalLM(nn.Module):
297
input_metadata)
298
return hidden_states
299
300
+ def compute_logits(self, hidden_states: torch.Tensor,
301
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
302
+ logits = self.logits_processor(self.model.embed_tokens.weight,
303
+ hidden_states, sampling_metadata)
304
+ return logits
305
+
306
def sample(
307
self,
300
- hidden_states: torch.Tensor,
308
+ logits: torch.Tensor,
309
sampling_metadata: SamplingMetadata,
310
) -> Optional[SamplerOutput]:
303
- next_tokens = self.sampler(self.model.embed_tokens.weight,
304
- hidden_states, sampling_metadata)
311
+ next_tokens = self.sampler(logits, sampling_metadata)
312
return next_tokens
313
314
def load_weights(self,
vllm/model_executor/models/gpt2.py
+11
-3
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
32
RowParallelLinear)
33
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding)
@@ -216,7 +217,8 @@ class GPT2LMHeadModel(nn.Module):
217
self.linear_method = linear_method
218
self.transformer = GPT2Model(config, linear_method)
219
self.lm_head_weight = self.transformer.wte.weight
219
- self.sampler = Sampler(config.vocab_size)
220
+ self.logits_processor = LogitsProcessor(config.vocab_size)
221
+ self.sampler = Sampler()
222
223
def forward(
224
self,
@@ -229,12 +231,18 @@ class GPT2LMHeadModel(nn.Module):
231
input_metadata)
232
return hidden_states
233
234
+ def compute_logits(self, hidden_states: torch.Tensor,
235
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
236
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
237
+ sampling_metadata)
238
+ return logits
239
+
240
def sample(
241
self,
234
- hidden_states: torch.Tensor,
242
+ logits: torch.Tensor,
243
sampling_metadata: SamplingMetadata,
244
) -> Optional[SamplerOutput]:
237
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
245
+ next_tokens = self.sampler(self.lm_head_weight, logits,
246
sampling_metadata)
247
return next_tokens
248
vllm/model_executor/models/gpt_bigcode.py
+11
-4
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
LinearMethodBase,
32
QKVParallelLinear,
33
RowParallelLinear)
34
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
@@ -237,7 +238,8 @@ class GPTBigCodeForCausalLM(nn.Module):
238
self.linear_method = linear_method
239
self.transformer = GPTBigCodeModel(config, linear_method)
240
self.lm_head_weight = self.transformer.wte.weight
240
- self.sampler = Sampler(config.vocab_size)
241
+ self.logits_processor = LogitsProcessor(config.vocab_size)
242
+ self.sampler = Sampler()
243
244
def forward(
245
self,
@@ -250,13 +252,18 @@ class GPTBigCodeForCausalLM(nn.Module):
252
input_metadata)
253
return hidden_states
254
255
+ def compute_logits(self, hidden_states: torch.Tensor,
256
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
257
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
258
+ sampling_metadata)
259
+ return logits
260
+
261
def sample(
262
self,
255
- hidden_states: torch.Tensor,
263
+ logits: torch.Tensor,
264
sampling_metadata: SamplingMetadata,
265
) -> Optional[SamplerOutput]:
258
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
259
- sampling_metadata)
266
+ next_tokens = self.sampler(logits, sampling_metadata)
267
return next_tokens
268
269
def load_weights(self,
vllm/model_executor/models/gpt_j.py
+11
-4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
QKVParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.rotary_embedding import get_rope
33
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding, ParallelLMHead)
@@ -224,7 +225,8 @@ class GPTJForCausalLM(nn.Module):
225
config.n_embd,
226
bias=True,
227
)
227
- self.sampler = Sampler(config.vocab_size)
228
+ self.logits_processor = LogitsProcessor(config.vocab_size)
229
+ self.sampler = Sampler()
230
231
def forward(
232
self,
@@ -237,13 +239,18 @@ class GPTJForCausalLM(nn.Module):
239
input_metadata)
240
return hidden_states
241
242
+ def compute_logits(self, hidden_states: torch.Tensor,
243
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
244
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
245
+ sampling_metadata, self.lm_head.bias)
246
+ return logits
247
+
248
def sample(
249
self,
242
- hidden_states: torch.Tensor,
250
+ logits: torch.Tensor,
251
sampling_metadata: SamplingMetadata,
252
) -> Optional[SamplerOutput]:
245
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
246
- sampling_metadata, self.lm_head.bias)
253
+ next_tokens = self.sampler(logits, sampling_metadata)
254
return next_tokens
255
256
def load_weights(self,
vllm/model_executor/models/gpt_neox.py
+11
-4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
QKVParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.rotary_embedding import get_rope
33
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding, ParallelLMHead)
@@ -238,7 +239,8 @@ class GPTNeoXForCausalLM(nn.Module):
239
config.vocab_size,
240
config.hidden_size,
241
)
241
- self.sampler = Sampler(config.vocab_size)
242
+ self.logits_processor = LogitsProcessor(config.vocab_size)
243
+ self.sampler = Sampler()
244
245
def forward(
246
self,
@@ -251,13 +253,18 @@ class GPTNeoXForCausalLM(nn.Module):
253
input_metadata)
254
return hidden_states
255
256
+ def compute_logits(self, hidden_states: torch.Tensor,
257
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
258
+ logits = self.logits_processor(self.embed_out.weight, hidden_states,
259
+ sampling_metadata)
260
+ return logits
261
+
262
def sample(
263
self,
256
- hidden_states: torch.Tensor,
264
+ logits: torch.Tensor,
265
sampling_metadata: SamplingMetadata,
266
) -> Optional[SamplerOutput]:
259
- next_tokens = self.sampler(self.embed_out.weight, hidden_states,
260
- sampling_metadata)
267
+ next_tokens = self.sampler(logits, sampling_metadata)
268
return next_tokens
269
270
def load_weights(self,
vllm/model_executor/models/internlm2.py
+11
-4
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
14
QKVParallelLinear,
15
RowParallelLinear)
16
from vllm.model_executor.layers.rotary_embedding import get_rope
17
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
VocabParallelEmbedding, ParallelLMHead)
@@ -250,7 +251,8 @@ class InternLM2ForCausalLM(nn.Module):
251
self.linear_method = linear_method
252
self.model = InternLM2Model(config, linear_method)
253
self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253
- self.sampler = Sampler(config.vocab_size)
254
+ self.logits_processor = LogitsProcessor(config.vocab_size)
255
+ self.sampler = Sampler()
256
257
def forward(
258
self,
@@ -263,13 +265,18 @@ class InternLM2ForCausalLM(nn.Module):
265
input_metadata)
266
return hidden_states
267
268
+ def compute_logits(self, hidden_states: torch.Tensor,
269
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
270
+ logits = self.logits_processor(self.output.weight, hidden_states,
271
+ sampling_metadata)
272
+ return logits
273
+
274
def sample(
275
self,
268
- hidden_states: torch.Tensor,
276
+ logits: torch.Tensor,
277
sampling_metadata: SamplingMetadata,
278
) -> Optional[SamplerOutput]:
271
- next_tokens = self.sampler(self.output.weight, hidden_states,
272
- sampling_metadata)
279
+ next_tokens = self.sampler(logits, sampling_metadata)
280
return next_tokens
281
282
def load_weights(self,
vllm/model_executor/models/llama.py
+14
-4
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
37
QKVParallelLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
@@ -325,7 +326,11 @@ class LlamaForCausalLM(nn.Module):
326
# compatibility
327
if not lora_config else lora_config.lora_vocab_padding_size,
328
)
328
- self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
329
+
330
+ logit_scale = getattr(config, "logit_scale", 1.0)
331
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
332
+ config.vocab_size, logit_scale)
333
+ self.sampler = Sampler()
334
335
def forward(
336
self,
@@ -338,13 +343,18 @@ class LlamaForCausalLM(nn.Module):
343
input_metadata)
344
return hidden_states
345
346
+ def compute_logits(self, hidden_states: torch.Tensor,
347
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
348
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
349
+ sampling_metadata)
350
+ return logits
351
+
352
def sample(
353
self,
343
- hidden_states: torch.Tensor,
354
+ logits: torch.Tensor,
355
sampling_metadata: SamplingMetadata,
356
) -> Optional[SamplerOutput]:
346
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
347
- sampling_metadata)
357
+ next_tokens = self.sampler(logits, sampling_metadata)
358
return next_tokens
359
360
def load_weights(self,
vllm/model_executor/models/mixtral.py
+12
-4
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
37
ReplicatedLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
@@ -369,7 +370,9 @@ class MixtralForCausalLM(nn.Module):
370
# compatibility
371
if not lora_config else lora_config.lora_vocab_padding_size,
372
)
372
- self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
373
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
374
+ config.vocab_size)
375
+ self.sampler = Sampler()
376
377
def forward(
378
self,
@@ -382,13 +385,18 @@ class MixtralForCausalLM(nn.Module):
385
input_metadata)
386
return hidden_states
387
388
+ def compute_logits(self, hidden_states: torch.Tensor,
389
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
390
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
391
+ sampling_metadata)
392
+ return logits
393
+
394
def sample(
395
self,
387
- hidden_states: Optional[torch.Tensor],
396
+ logits: Optional[torch.Tensor],
397
sampling_metadata: SamplingMetadata,
398
) -> Optional[SamplerOutput]:
390
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
391
- sampling_metadata)
399
+ next_tokens = self.sampler(logits, sampling_metadata)
400
return next_tokens
401
402
def load_weights(self,
vllm/model_executor/models/mixtral_quant.py
+11
-4
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
39
QKVParallelLinear,
40
RowParallelLinear)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
VocabParallelEmbedding, ParallelLMHead)
@@ -344,7 +345,8 @@ class MixtralForCausalLM(nn.Module):
345
self.linear_method = linear_method
346
self.model = MixtralModel(config, linear_method)
347
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
347
- self.sampler = Sampler(config.vocab_size)
348
+ self.logits_processor = LogitsProcessor(config.vocab_size)
349
+ self.sampler = Sampler()
350
351
def forward(
352
self,
@@ -357,13 +359,18 @@ class MixtralForCausalLM(nn.Module):
359
input_metadata)
360
return hidden_states
361
362
+ def compute_logits(self, hidden_states: torch.Tensor,
363
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
364
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
365
+ sampling_metadata)
366
+ return logits
367
+
368
def sample(
369
self,
362
- hidden_states: Optional[torch.Tensor],
370
+ logits: Optional[torch.Tensor],
371
sampling_metadata: SamplingMetadata,
372
) -> Optional[SamplerOutput]:
365
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
366
- sampling_metadata)
373
+ next_tokens = self.sampler(logits, sampling_metadata)
374
return next_tokens
375
376
def load_weights(self,
vllm/model_executor/models/mpt.py
+11
-4
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
LinearMethodBase,
14
QKVParallelLinear,
15
RowParallelLinear)
16
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
from vllm.model_executor.layers.sampler import Sampler
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
VocabParallelEmbedding)
@@ -259,7 +260,8 @@ class MPTForCausalLM(nn.Module):
260
261
self.transformer = MPTModel(config, linear_method)
262
self.lm_head_weight = self.transformer.wte.weight
262
- self.sampler = Sampler(config.vocab_size)
263
+ self.logits_processor = LogitsProcessor(config.vocab_size)
264
+ self.sampler = Sampler()
265
266
def forward(
267
self,
@@ -272,13 +274,18 @@ class MPTForCausalLM(nn.Module):
274
input_metadata)
275
return hidden_states
276
277
+ def compute_logits(self, hidden_states: torch.Tensor,
278
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
279
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
280
+ sampling_metadata)
281
+ return logits
282
+
283
def sample(
284
self,
277
- hidden_states: torch.Tensor,
285
+ logits: torch.Tensor,
286
sampling_metadata: SamplingMetadata,
287
) -> Optional[SamplerOutput]:
280
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
281
- sampling_metadata)
288
+ next_tokens = self.sampler(logits, sampling_metadata)
289
return next_tokens
290
291
def load_weights(self,
vllm/model_executor/models/neuron/llama.py
+11
-4
@@ -7,6 +7,7 @@ from torch import nn
7
from transformers import LlamaConfig
8
9
from vllm.model_executor.input_metadata import InputMetadata
10
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
11
from vllm.model_executor.layers.sampler import Sampler
12
from vllm.model_executor.sampling_metadata import SamplingMetadata
13
from vllm.sequence import SamplerOutput
@@ -25,7 +26,8 @@ class LlamaForCausalLM(nn.Module):
26
self.config = config
27
self.linear_method = linear_method
28
self.model = None
28
- self.sampler = Sampler(config.vocab_size)
29
+ self.logits_processor = LogitsProcessor(config.vocab_size)
30
+ self.sampler = Sampler()
31
32
def forward(
33
self,
@@ -45,13 +47,18 @@ class LlamaForCausalLM(nn.Module):
47
start_ids=seq_ids.flatten())
48
return logits
49
50
+ def compute_logits(self, hidden_states: torch.Tensor,
51
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
52
+ logits = self.logits_processor(self.model.chkpt_model.lm_head,
53
+ hidden_states, sampling_metadata)
54
+ return logits
55
+
56
def sample(
57
self,
50
- hidden_states: torch.Tensor,
58
+ logits: torch.Tensor,
59
sampling_metadata: SamplingMetadata,
60
) -> Optional[SamplerOutput]:
53
- next_tokens = self.sampler(self.model.chkpt_model.lm_head,
54
- hidden_states, sampling_metadata)
61
+ next_tokens = self.sampler(logits, sampling_metadata)
62
return next_tokens
63
64
def load_weights(self,
vllm/model_executor/models/neuron/mistral.py
+11
-4
@@ -6,6 +6,7 @@ from torch import nn
6
from transformers import MistralConfig
7
8
from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
10
from vllm.model_executor.layers.sampler import Sampler
11
from vllm.model_executor.sampling_metadata import SamplingMetadata
12
from vllm.sequence import SamplerOutput
@@ -26,7 +27,8 @@ class MistralForCausalLM(nn.Module):
27
self.linear_method = linear_method
28
self.model = None
29
self.lm_head = None
29
- self.sampler = Sampler(config.vocab_size)
30
+ self.logits_processor = LogitsProcessor(config.vocab_size)
31
+ self.sampler = Sampler()
32
33
def forward(
34
self,
@@ -48,13 +50,18 @@ class MistralForCausalLM(nn.Module):
50
start_ids=seq_ids)
51
return logits
52
53
+ def compute_logits(self, hidden_states: torch.Tensor,
54
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
55
+ logits = self.logits_processor(self.model.chkpt_model.lm_head,
56
+ hidden_states, sampling_metadata)
57
+ return logits
58
+
59
def sample(
60
self,
53
- hidden_states: torch.Tensor,
61
+ logits: torch.Tensor,
62
sampling_metadata: SamplingMetadata,
63
) -> Optional[SamplerOutput]:
56
- next_tokens = self.sampler(self.model.chkpt_model.lm_head,
57
- hidden_states, sampling_metadata)
64
+ next_tokens = self.sampler(logits, sampling_metadata)
65
return next_tokens
66
67
def load_weights(self,
vllm/model_executor/models/olmo.py
+11
-4
@@ -51,6 +51,7 @@ from vllm.model_executor.layers.linear import (
51
RowParallelLinear,
52
)
53
from vllm.model_executor.layers.rotary_embedding import get_rope
54
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
55
from vllm.model_executor.layers.sampler import Sampler
56
from vllm.model_executor.layers.vocab_parallel_embedding import (
57
VocabParallelEmbedding)
@@ -336,7 +337,8 @@ class OLMoForCausalLM(nn.Module):
337
self.lm_head_weight = (self.model.transformer.wte.weight
338
if config.weight_tying else
339
self.model.transformer.ff_out.weight)
339
- self.sampler = Sampler(config.vocab_size)
340
+ self.logits_processor = LogitsProcessor(config.vocab_size)
341
+ self.sampler = Sampler()
342
343
def forward(
344
self,
@@ -353,13 +355,18 @@ class OLMoForCausalLM(nn.Module):
355
)
356
return hidden_states
357
358
+ def compute_logits(self, hidden_states: torch.Tensor,
359
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
360
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
361
+ sampling_metadata)
362
+ return logits
363
+
364
def sample(
365
self,
358
- hidden_states: torch.Tensor,
366
+ logits: torch.Tensor,
367
sampling_metadata: SamplingMetadata,
368
) -> Optional[SamplerOutput]:
361
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
362
- sampling_metadata)
369
+ next_tokens = self.sampler(logits, sampling_metadata)
370
return next_tokens
371
372
def load_weights(
vllm/model_executor/models/opt.py
+11
-4
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
QKVParallelLinear,
32
ReplicatedLinear,
33
RowParallelLinear)
34
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
@@ -292,7 +293,8 @@ class OPTForCausalLM(nn.Module):
293
self.linear_method = linear_method
294
self.model = OPTModel(config, linear_method)
295
self.lm_head_weight = self.model.decoder.embed_tokens.weight
295
- self.sampler = Sampler(config.vocab_size)
296
+ self.logits_processor = LogitsProcessor(config.vocab_size)
297
+ self.sampler = Sampler()
298
299
def forward(
300
self,
@@ -305,13 +307,18 @@ class OPTForCausalLM(nn.Module):
307
input_metadata)
308
return hidden_states
309
310
+ def compute_logits(self, hidden_states: torch.Tensor,
311
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
312
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
313
+ sampling_metadata)
314
+ return logits
315
+
316
def sample(
317
self,
310
- hidden_states: torch.Tensor,
318
+ logits: torch.Tensor,
319
sampling_metadata: SamplingMetadata,
320
) -> Optional[SamplerOutput]:
313
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
314
- sampling_metadata)
321
+ next_tokens = self.sampler(logits, sampling_metadata)
322
return next_tokens
323
324
def load_weights(self,
vllm/model_executor/models/orion.py
+11
-4
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
18
QKVParallelLinear,
19
RowParallelLinear)
20
from vllm.model_executor.layers.rotary_embedding import get_rope
21
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
22
from vllm.model_executor.layers.sampler import Sampler
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
VocabParallelEmbedding, ParallelLMHead)
@@ -256,7 +257,8 @@ class OrionForCausalLM(nn.Module):
257
self.linear_method = linear_method
258
self.model = OrionModel(config, linear_method)
259
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
259
- self.sampler = Sampler(config.vocab_size)
260
+ self.logits_processor = LogitsProcessor(config.vocab_size)
261
+ self.sampler = Sampler()
262
263
def forward(
264
self,
@@ -269,13 +271,18 @@ class OrionForCausalLM(nn.Module):
271
input_metadata)
272
return hidden_states
273
274
+ def compute_logits(self, hidden_states: torch.Tensor,
275
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
276
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
277
+ sampling_metadata)
278
+ return logits
279
+
280
def sample(
281
self,
274
- hidden_states: torch.Tensor,
282
+ logits: torch.Tensor,
283
sampling_metadata: SamplingMetadata,
284
) -> Optional[SamplerOutput]:
277
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
278
- sampling_metadata)
285
+ next_tokens = self.sampler(logits, sampling_metadata)
286
return next_tokens
287
288
def load_weights(self,
vllm/model_executor/models/phi.py
+11
-5
@@ -49,6 +49,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49
QKVParallelLinear,
50
RowParallelLinear)
51
from vllm.model_executor.layers.rotary_embedding import get_rope
52
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
53
from vllm.model_executor.layers.sampler import Sampler
54
from vllm.model_executor.layers.vocab_parallel_embedding import (
55
VocabParallelEmbedding, ParallelLMHead)
@@ -240,7 +241,8 @@ class PhiForCausalLM(nn.Module):
241
self.lm_head = ParallelLMHead(config.vocab_size,
242
config.hidden_size,
243
bias=True)
243
- self.sampler = Sampler(config.vocab_size)
244
+ self.logits_processor = LogitsProcessor(config.vocab_size)
245
+ self.sampler = Sampler()
246
247
def forward(
248
self,
@@ -254,14 +256,18 @@ class PhiForCausalLM(nn.Module):
256
257
return hidden_states
258
259
+ def compute_logits(self, hidden_states: torch.Tensor,
260
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
261
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
262
+ sampling_metadata, self.lm_head.bias)
263
+ return logits
264
+
265
def sample(
266
self,
259
- hidden_states: torch.Tensor,
267
+ logits: torch.Tensor,
268
sampling_metadata: SamplingMetadata,
269
) -> Optional[SamplerOutput]:
262
- head = self.lm_head
263
- next_tokens = self.sampler(head.weight, hidden_states,
264
- sampling_metadata, head.bias)
270
+ next_tokens = self.sampler(logits, sampling_metadata)
271
return next_tokens
272
273
def load_weights(self,
vllm/model_executor/models/qwen.py
+11
-4
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
19
QKVParallelLinear,
20
RowParallelLinear)
21
from vllm.model_executor.layers.rotary_embedding import get_rope
22
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
23
from vllm.model_executor.layers.sampler import Sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
VocabParallelEmbedding, ParallelLMHead)
@@ -230,7 +231,8 @@ class QWenLMHeadModel(nn.Module):
231
self.linear_method = linear_method
232
self.transformer = QWenModel(config, linear_method)
233
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
233
- self.sampler = Sampler(config.vocab_size)
234
+ self.logits_processor = LogitsProcessor(config.vocab_size)
235
+ self.sampler = Sampler()
236
237
def forward(
238
self,
@@ -243,13 +245,18 @@ class QWenLMHeadModel(nn.Module):
245
input_metadata)
246
return hidden_states
247
248
+ def compute_logits(self, hidden_states: torch.Tensor,
249
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
250
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
251
+ sampling_metadata)
252
+ return logits
253
+
254
def sample(
255
self,
248
- hidden_states: torch.Tensor,
256
+ logits: torch.Tensor,
257
sampling_metadata: SamplingMetadata,
258
) -> Optional[SamplerOutput]:
251
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252
- sampling_metadata)
259
+ next_tokens = self.sampler(logits, sampling_metadata)
260
return next_tokens
261
262
def load_weights(self,
vllm/model_executor/models/qwen2.py
+15
-9
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
37
QKVParallelLinear,
38
RowParallelLinear)
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
VocabParallelEmbedding, ParallelLMHead)
@@ -300,11 +301,15 @@ class Qwen2ForCausalLM(nn.Module):
301
self.linear_method = linear_method
302
self.model = Qwen2Model(config, linear_method)
303
303
- if not config.tie_word_embeddings:
304
+ if config.tie_word_embeddings:
305
+ self.lm_head_weight = self.model.embed_tokens.weight
306
+ else:
307
self.lm_head = ParallelLMHead(config.vocab_size,
308
config.hidden_size)
309
+ self.lm_head_weight = self.lm_head.weight
310
307
- self.sampler = Sampler(config.vocab_size)
311
+ self.logits_processor = LogitsProcessor(config.vocab_size)
312
+ self.sampler = Sampler()
313
314
def forward(
315
self,
@@ -317,17 +322,18 @@ class Qwen2ForCausalLM(nn.Module):
322
input_metadata)
323
return hidden_states
324
325
+ def compute_logits(self, hidden_states: torch.Tensor,
326
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
327
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
328
+ sampling_metadata)
329
+ return logits
330
+
331
def sample(
332
self,
322
- hidden_states: torch.Tensor,
333
+ logits: torch.Tensor,
334
sampling_metadata: SamplingMetadata,
335
) -> Optional[SamplerOutput]:
325
- if self.config.tie_word_embeddings:
326
- lm_head_weight = self.model.embed_tokens.weight
327
- else:
328
- lm_head_weight = self.lm_head.weight
329
- next_tokens = self.sampler(lm_head_weight, hidden_states,
330
- sampling_metadata)
336
+ next_tokens = self.sampler(logits, sampling_metadata)
337
return next_tokens
338
339
def load_weights(self,
vllm/model_executor/models/stablelm.py
+11
-4
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
33
QKVParallelLinear,
34
RowParallelLinear)
35
from vllm.model_executor.layers.rotary_embedding import get_rope
36
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding, ParallelLMHead)
@@ -238,7 +239,8 @@ class StablelmForCausalLM(nn.Module):
239
self.linear_method = linear_method
240
self.model = StableLMEpochModel(config, linear_method)
241
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241
- self.sampler = Sampler(config.vocab_size)
242
+ self.logits_processor = LogitsProcessor(config.vocab_size)
243
+ self.sampler = Sampler()
244
245
def forward(
246
self,
@@ -251,13 +253,18 @@ class StablelmForCausalLM(nn.Module):
253
input_metadata)
254
return hidden_states
255
256
+ def compute_logits(self, hidden_states: torch.Tensor,
257
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
258
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
259
+ sampling_metadata)
260
+ return logits
261
+
262
def sample(
263
self,
256
- hidden_states: torch.Tensor,
264
+ logits: torch.Tensor,
265
sampling_metadata: SamplingMetadata,
266
) -> Optional[SamplerOutput]:
259
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260
- sampling_metadata)
267
+ next_tokens = self.sampler(logits, sampling_metadata)
268
return next_tokens
269
270
def load_weights(self,
vllm/model_executor/models/starcoder2.py
+12
-4
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
LinearMethodBase,
33
QKVParallelLinear,
34
RowParallelLinear)
35
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
@@ -254,7 +255,9 @@ class Starcoder2ForCausalLM(nn.Module):
255
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
256
)
257
self.lm_head_weight = self.lm_head.weight
257
- self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
258
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
259
+ config.vocab_size)
260
+ self.sampler = Sampler()
261
262
def forward(
263
self,
@@ -267,13 +270,18 @@ class Starcoder2ForCausalLM(nn.Module):
270
input_metadata)
271
return hidden_states
272
273
+ def compute_logits(self, hidden_states: torch.Tensor,
274
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
275
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
276
+ sampling_metadata)
277
+ return logits
278
+
279
def sample(
280
self,
272
- hidden_states: Optional[torch.Tensor],
281
+ logits: Optional[torch.Tensor],
282
sampling_metadata: SamplingMetadata,
283
) -> Optional[SamplerOutput]:
275
- next_tokens = self.sampler(self.lm_head_weight, hidden_states,
276
- sampling_metadata)
284
+ next_tokens = self.sampler(logits, sampling_metadata)
285
return next_tokens
286
287
def load_weights(self,
vllm/worker/model_runner.py
+8
-1
@@ -613,9 +613,16 @@ class ModelRunner:
613
input_metadata=input_metadata,
614
)
615
616
+ # Compute the logits.
617
+ logits = self.model.compute_logits(hidden_states, sampling_metadata)
618
+
619
+ # Only perform sampling in the driver worker.
620
+ if not sampling_metadata.perform_sampling:
621
+ return None
622
+
623
# Sample the next token.
624
output = self.model.sample(
618
- hidden_states=hidden_states,
625
+ logits=logits,
626
sampling_metadata=sampling_metadata,
627
)
628
return output