@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f1c0fc3919
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Migrate `logits` computation and gather to `model_runner` (#3233)

Roy committed Mar 21, 2024 at 07:25 UTC f1c0fc391909e55fce5f109893f3c483f69a091f
35 files changed +577 -306
.buildkite/test-pipeline.yaml
+3
@@ -49,6 +49,9 @@ steps:
49 - label: Samplers Test
50 command: pytest -v -s samplers
51
52 +- label: LogitsProcessor Test
53 + command: pytest -v -s test_logits_processor.py
54 +
55 - label: Worker Test
56 command: pytest -v -s worker
57
tests/lora/conftest.py
+5 -2
@@ -13,6 +13,7 @@ from huggingface_hub import snapshot_download
13 import vllm
14 from vllm.config import LoRAConfig
15 from vllm.model_executor.layers.sampler import Sampler
16 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 from vllm.model_executor.model_loader import get_model
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 MergedColumnParallelLinear,
@@ -85,7 +86,8 @@ def dummy_model() -> nn.Module:
86 ("outact", nn.Sigmoid()),
87 # Special handling for lm_head & sampler
88 ("lm_head", ParallelLMHead(512, 10)),
88 - ("sampler", Sampler(512))
89 + ("logits_processor", LogitsProcessor(512)),
90 + ("sampler", Sampler())
91 ]))
92 model.config = MagicMock()
93 return model
@@ -110,7 +112,8 @@ def dummy_model_gate_up() -> nn.Module:
112 ("outact", nn.Sigmoid()),
113 # Special handling for lm_head & sampler
114 ("lm_head", ParallelLMHead(512, 10)),
113 - ("sampler", Sampler(512))
115 + ("logits_processor", LogitsProcessor(512)),
116 + ("sampler", Sampler())
117 ]))
118 model.config = MagicMock()
119 return model
tests/lora/test_layers.py
+36 -30
@@ -13,14 +13,14 @@ from vllm.lora.layers import (
13 QKVParallelLinearWithLora,
14 VocabParallelEmbeddingWithLoRA,
15 RowParallelLinearWithLoRA,
16 - SamplerWithLoRA,
16 + LogitsProcessorWithLoRA,
17 LoRAMapping,
18 BaseLayerWithLoRA,
19 )
20 from vllm.lora.models import (LoRALayerWeights, convert_mapping,
21 PackedLoRALayerWeights)
22 from vllm.config import LoRAConfig
23 -from vllm.model_executor.layers.sampler import Sampler
23 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
24 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25 MergedColumnParallelLinear,
26 RowParallelLinear,
@@ -394,7 +394,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
394 @torch.inference_mode()
395 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
396 @pytest.mark.parametrize("device", CUDA_DEVICES)
397 -def test_lm_head_sampler(dist_init, num_loras, device) -> None:
397 +def test_lm_head_logits_processor(dist_init, num_loras, device) -> None:
398
399 torch.set_default_device(device)
400 max_loras = 8
@@ -402,28 +402,29 @@ def test_lm_head_sampler(dist_init, num_loras, device) -> None:
402 max_lora_rank=8,
403 lora_dtype=torch.float16)
404
405 - def create_random_sampler_layer():
405 + def _pretest():
406 linear = ParallelLMHead(32000 + lora_config.lora_extra_vocab_size,
407 1024, 32000)
408 linear.weight.data = torch.rand_like(linear.weight.data)
409 linear.weight.data[:, 32000:] = 0
410 - sampler = Sampler(32000 + lora_config.lora_extra_vocab_size, 32000)
411 - lora_sampler = SamplerWithLoRA(sampler, 1024, linear.weight.dtype,
412 - linear.weight.device)
413 - lora_sampler.create_lora_weights(max_loras, lora_config)
410 + logits_processor = LogitsProcessor(
411 + 32000 + lora_config.lora_extra_vocab_size, 32000)
412 + lora_logits_processor = LogitsProcessorWithLoRA(
413 + logits_processor, 1024, linear.weight.dtype, linear.weight.device)
414 + lora_logits_processor.create_lora_weights(max_loras, lora_config)
415
415 - return linear, sampler, lora_sampler
416 + return linear, logits_processor, lora_logits_processor
417
418 for i in range(10):
419 set_random_seed(i)
420
421 id_to_index = get_random_id_to_index(num_loras, max_loras)
421 - linear, sampler, lora_sampler = create_random_sampler_layer()
422 + linear, logits_processor, lora_logits_processor = _pretest()
423
424 # NOTE: all the generated loras share the same embeddings tensor.
425 lora_dict, _ = populate_loras(
426 id_to_index,
426 - layer=lora_sampler,
427 + layer=lora_logits_processor,
428 layer_weights=linear.weight,
429 generate_embeddings_tensor=1024,
430 )
@@ -447,34 +448,37 @@ def test_lm_head_sampler(dist_init, num_loras, device) -> None:
448 32000,
449 lora_config.lora_extra_vocab_size,
450 )
450 - lora_sampler.set_mapping(*mapping_info, )
451 + lora_logits_processor.set_mapping(*mapping_info, )
452
452 - lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
453 - embedding=linear.weight,
454 - embedding_bias=None)
453 + lora_result = lora_logits_processor._get_logits(
454 + hidden_states=torch.cat(inputs),
455 + embedding=linear.weight,
456 + embedding_bias=None)
457
458 original_weight = linear.weight.clone()
459
458 - linear.weight[sampler.org_vocab_size:sampler.org_vocab_size +
460 + linear.weight[logits_processor.
461 + org_vocab_size:logits_processor.org_vocab_size +
462 embeddings_tensor_len] = embeddings_tensor
463
461 - sampler.org_vocab_size = 32000 + lora_config.lora_extra_vocab_size
464 + logits_processor.org_vocab_size = (32000 +
465 + lora_config.lora_extra_vocab_size)
466 expected_results = []
467 for input_, lora_id in zip(inputs, prompt_mapping):
468 lora = lora_dict[lora_id]
465 - result = sampler._get_logits(hidden_states=input_,
466 - embedding=linear.weight,
467 - embedding_bias=None)
469 + result = logits_processor._get_logits(hidden_states=input_,
470 + embedding=linear.weight,
471 + embedding_bias=None)
472 result[:, 32000 + embeddings_tensor_len:] = float("-inf")
473 result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
474 expected_results.append(result)
475 expected_result = torch.cat(expected_results)
472 - sampler.org_vocab_size = 32000
476 + logits_processor.org_vocab_size = 32000
477
478 # Check that resetting the lora weights succeeds
479
480 for slot_idx in range(max_loras):
477 - lora_sampler.reset_lora(slot_idx)
481 + lora_logits_processor.reset_lora(slot_idx)
482
483 inputs, index_mapping, prompt_mapping = create_random_inputs(
484 active_lora_ids=[0],
@@ -488,14 +492,16 @@ def test_lm_head_sampler(dist_init, num_loras, device) -> None:
492 mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
493 32000,
494 lora_config.lora_extra_vocab_size)
491 - lora_sampler.set_mapping(*mapping_info, )
492 -
493 - lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
494 - embedding=original_weight,
495 - embedding_bias=None)[:, :32000]
496 - expected_result = sampler._get_logits(hidden_states=torch.cat(inputs),
497 - embedding=original_weight,
498 - embedding_bias=None)
495 + lora_logits_processor.set_mapping(*mapping_info, )
496 +
497 + lora_result = lora_logits_processor._get_logits(
498 + hidden_states=torch.cat(inputs),
499 + embedding=original_weight,
500 + embedding_bias=None)[:, :32000]
501 + expected_result = logits_processor._get_logits(
502 + hidden_states=torch.cat(inputs),
503 + embedding=original_weight,
504 + embedding_bias=None)
505
506 rtol, atol = TOLERANCES[lora_result.dtype]
507 assert torch.allclose(lora_result,
tests/samplers/test_sampler.py
+20 -75
@@ -15,17 +15,12 @@ from vllm.worker.model_runner import ModelRunner
15
16 class MockLogitsSampler(Sampler):
17
18 - def __init__(self, vocab_size: int, fake_logits: torch.Tensor):
19 - super().__init__(vocab_size=vocab_size)
18 + def __init__(self, fake_logits: torch.Tensor):
19 + super().__init__()
20 self.fake_logits = fake_logits
21
22 def forward(self, *args, **kwargs):
23 - with patch(
24 - "vllm.model_executor.layers.sampler._prune_hidden_states",
25 - lambda x, y: x), patch(
26 - "vllm.model_executor.layers.sampler.Sampler._get_logits",
27 - lambda *args, **kwargs: self.fake_logits):
28 - return super().forward(*args, **kwargs)
23 + return super().forward(*args, **kwargs)
24
25
26 def _prepare_test(
@@ -36,7 +31,7 @@ def _prepare_test(
31 fake_logits = torch.full((batch_size, vocab_size),
32 1e-2,
33 dtype=input_tensor.dtype)
39 - sampler = MockLogitsSampler(32000, fake_logits)
34 + sampler = MockLogitsSampler(fake_logits)
35 model_runner = ModelRunner(None, None, None, None, None)
36 return input_tensor, fake_logits, sampler, model_runner
37
@@ -70,9 +65,7 @@ def _do_sample(
65 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
66 prompt_lens,
67 subquery_lens=prompt_lens)
73 - return sampler(embedding=None,
74 - hidden_states=input_tensor,
75 - sampling_metadata=sampling_metadata)
68 + return sampler(logits=input_tensor, sampling_metadata=sampling_metadata)
69
70
71 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
@@ -85,8 +78,8 @@ def test_sampler_all_greedy(seed: int, device: str):
78 batch_size)
79
80 sampling_params = SamplingParams(temperature=0)
88 - sampler_output = _do_sample(batch_size, input_tensor, sampler,
89 - model_runner, sampling_params)
81 + sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
82 + sampling_params)
83 expected = torch.argmax(fake_logits, dim=-1)
84 for i, sequence_output in enumerate(sampler_output):
85 for nth_output in sequence_output.samples:
@@ -111,8 +104,8 @@ def test_sampler_all_random(seed: int, device: str):
104 temperature=1.0,
105 n=random.randint(1, 10),
106 )
114 - sampler_output = _do_sample(batch_size, input_tensor, sampler,
115 - model_runner, sampling_params)
107 + sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
108 + sampling_params)
109
110 for i, sequence_output in enumerate(sampler_output):
111 for nth_output in sequence_output.samples:
@@ -127,8 +120,7 @@ def test_sampler_all_random_seed(seed: int, device: str):
120 set_random_seed(seed)
121 torch.set_default_device(device)
122 batch_size = random.randint(1, 256)
130 - input_tensor, fake_logits, sampler, model_runner = _prepare_test(
131 - batch_size)
123 + _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
124
125 for i in range(batch_size):
126 fake_logits[i, i] = 1e2
@@ -138,8 +130,8 @@ def test_sampler_all_random_seed(seed: int, device: str):
130 n=random.randint(1, 10),
131 seed=random.randint(0, 10000),
132 )
141 - sampler_output = _do_sample(batch_size, input_tensor, sampler,
142 - model_runner, sampling_params)
133 + sampler_output = _do_sample(batch_size, fake_logits, sampler, model_runner,
134 + sampling_params)
135
136 for i, sequence_output in enumerate(sampler_output):
137 for nth_output in sequence_output.samples:
@@ -154,18 +146,17 @@ def test_sampler_all_random_seed_deterministic(seed: int, device: str):
146 set_random_seed(seed)
147 torch.set_default_device(device)
148 batch_size = random.randint(1, 256)
157 - input_tensor, fake_logits, sampler, model_runner = _prepare_test(
158 - batch_size)
149 + _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
150
151 sampling_params = SamplingParams(
152 temperature=1.0,
153 n=random.randint(1, 10),
154 seed=random.randint(0, 10000),
155 )
165 - first_sampler_output = _do_sample(batch_size, input_tensor, sampler,
156 + first_sampler_output = _do_sample(batch_size, fake_logits, sampler,
157 model_runner, sampling_params)
158
168 - second_sampler_output = _do_sample(batch_size, input_tensor, sampler,
159 + second_sampler_output = _do_sample(batch_size, fake_logits, sampler,
160 model_runner, sampling_params)
161
162 assert first_sampler_output == second_sampler_output
@@ -179,15 +170,14 @@ def test_sampler_all_beam(seed: int, device: str):
170 set_random_seed(seed)
171 torch.set_default_device(device)
172 batch_size = random.randint(1, 256)
182 - input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
173 + _, fake_logits, sampler, model_runner = _prepare_test(batch_size)
174
175 sampling_params = SamplingParams(
176 temperature=0,
177 best_of=2,
178 use_beam_search=True,
179 )
189 - _do_sample(batch_size, input_tensor, sampler, model_runner,
190 - sampling_params)
180 + _do_sample(batch_size, fake_logits, sampler, model_runner, sampling_params)
181 # no assertion here as I am not sure how to determine whether
182 # the outputs are expected - in other words, this just tests
183 # whether there are no exceptions in the sampler
@@ -246,8 +236,7 @@ def test_sampler_mixed(seed: int, device: str):
236 def test_sampling(model_runner: ModelRunner):
237 sampling_metadata = model_runner._prepare_sample(
238 seq_group_metadata_list, prompt_lens, subquery_lens=prompt_lens)
249 - sampler_output = sampler(embedding=None,
250 - hidden_states=input_tensor,
239 + sampler_output = sampler(logits=fake_logits,
240 sampling_metadata=sampling_metadata)
241
242 for i, (sequence_output, metadata) in enumerate(
@@ -294,48 +283,6 @@ def test_sampler_mixed(seed: int, device: str):
283 del model_runner
284
285
297 -@pytest.mark.parametrize("seed", RANDOM_SEEDS)
298 -@pytest.mark.parametrize("device", CUDA_DEVICES)
299 -def test_sampler_logits_processors(seed: int, device: str):
300 - set_random_seed(seed)
301 - torch.set_default_device(device)
302 - batch_size = random.randint(1, 256)
303 - input_tensor, _, sampler, model_runner = _prepare_test(batch_size)
304 -
305 - # This sample logits processor gives maximum score to the i-th token,
306 - # where i is the length of the input sequence.
307 - # We therefore expect the output token sequence to be [0, 1, 2, ...]
308 - def pick_ith(token_ids, logits):
309 - logits[len(token_ids)] = torch.finfo(logits.dtype).max
310 - return logits
311 -
312 - seq_group_metadata_list = []
313 - prompt_lens = []
314 - for i in range(batch_size):
315 - seq_group_metadata_list.append(
316 - SequenceGroupMetadata(
317 - request_id=f"test_{i}",
318 - is_prompt=True,
319 - seq_data={0: SequenceData([1, 2, 3])},
320 - sampling_params=SamplingParams(temperature=0,
321 - logits_processors=[pick_ith]),
322 - block_tables={0: [1]},
323 - ))
324 - prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
325 -
326 - sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
327 - prompt_lens,
328 - subquery_lens=prompt_lens)
329 - sampler_output = sampler(embedding=None,
330 - hidden_states=input_tensor,
331 - sampling_metadata=sampling_metadata)
332 - for _, sequence_output in enumerate(sampler_output):
333 - for idx, nth_output in enumerate(sequence_output.samples):
334 - assert nth_output.output_token == idx
335 -
336 - del model_runner
337 -
338 -
286 @pytest.mark.parametrize("seed", RANDOM_SEEDS)
287 @pytest.mark.parametrize("device", CUDA_DEVICES)
288 def test_sampler_top_k_top_p(seed: int, device: str):
@@ -352,7 +299,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
299 size=(batch_size, vocab_size),
300 device=input_tensor.device,
301 dtype=input_tensor.dtype)
355 - sampler = MockLogitsSampler(32000, fake_logits)
302 + sampler = MockLogitsSampler(fake_logits)
303 model_runner = ModelRunner(None, None, None, None, None)
304
305 generation_model = GenerationMixin()
@@ -391,9 +338,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
338 return [[prob.topk(1, dim=-1).indices.tolist(), [0]] for prob in probs]
339
340 with patch("vllm.model_executor.layers.sampler._sample", mock_sample):
394 - sampler(embedding=None,
395 - hidden_states=input_tensor,
396 - sampling_metadata=sampling_metadata)
341 + sampler(logits=fake_logits, sampling_metadata=sampling_metadata)
342 hf_probs = warpers(torch.zeros_like(fake_logits), fake_logits.clone())
343 hf_probs = torch.softmax(hf_probs, dim=-1, dtype=torch.float)
344 assert torch.allclose(hf_probs, sample_probs, atol=1e-5)
tests/test_logits_processor.py new
+94
@@ -0,0 +1,94 @@
1 +import random
2 +from typing import Tuple
3 +from unittest.mock import patch
4 +
5 +import pytest
6 +import torch
7 +
8 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 +from vllm.model_executor.utils import set_random_seed
10 +from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11 +from vllm.worker.model_runner import ModelRunner
12 +
13 +
14 +class MockLogitsProcessor(LogitsProcessor):
15 +
16 + def __init__(self, vocab_size: int, scale: float,
17 + fake_logits: torch.Tensor):
18 + super().__init__(vocab_size=vocab_size, scale=scale)
19 + self.fake_logits = fake_logits.clone()
20 +
21 + def forward(self, *args, **kwargs):
22 + with patch(
23 + "vllm.model_executor.layers.logits_processor._prune_hidden_states",
24 + lambda x, y: x
25 + ), patch(
26 + "vllm.model_executor.layers.logits_processor.LogitsProcessor._get_logits",
27 + lambda *args, **kwargs: self.fake_logits):
28 + return super().forward(*args, **kwargs)
29 +
30 +
31 +def _prepare_test(
32 + batch_size: int
33 +) -> Tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor, ModelRunner]:
34 + vocab_size = 32000
35 + input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
36 + fake_logits = torch.full((batch_size, vocab_size),
37 + 1e-2,
38 + dtype=input_tensor.dtype)
39 + logits_processor = MockLogitsProcessor(32000, 0.5, fake_logits)
40 + model_runner = ModelRunner(None, None, None, None, None)
41 + return input_tensor, fake_logits, logits_processor, model_runner
42 +
43 +
44 +RANDOM_SEEDS = list(range(128))
45 +CUDA_DEVICES = [
46 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
47 +]
48 +
49 +
50 +@pytest.mark.parametrize("seed", RANDOM_SEEDS)
51 +@pytest.mark.parametrize("device", CUDA_DEVICES)
52 +def test_logits_processors(seed: int, device: str):
53 + set_random_seed(seed)
54 + torch.set_default_device(device)
55 + batch_size = random.randint(1, 256)
56 + input_tensor, fake_logits, logits_processor, model_runner = _prepare_test(
57 + batch_size)
58 +
59 + # This sample logits processor gives infinite score to the i-th token,
60 + # where i is the length of the input sequence.
61 + # We therefore expect the output token sequence to be [0, 1, 2, ...]
62 + def pick_ith(token_ids, logits):
63 + logits[len(token_ids)] = float("inf")
64 + return logits
65 +
66 + seq_group_metadata_list = []
67 + prompt_lens = []
68 + for i in range(batch_size):
69 + seq_group_metadata_list.append(
70 + SequenceGroupMetadata(
71 + request_id=f"test_{i}",
72 + is_prompt=True,
73 + seq_data={0: SequenceData([1, 2, 3])},
74 + sampling_params=SamplingParams(temperature=0,
75 + logits_processors=[pick_ith]),
76 + block_tables={0: [1]},
77 + ))
78 + prompt_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
79 +
80 + sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
81 + prompt_lens,
82 + subquery_lens=prompt_lens)
83 + logits_processor_output = logits_processor(
84 + embedding=None,
85 + hidden_states=input_tensor,
86 + sampling_metadata=sampling_metadata)
87 +
88 + assert torch.isinf(logits_processor_output[:, 0]).all()
89 +
90 + fake_logits *= logits_processor.scale
91 + assert torch.allclose(logits_processor_output[:, 1], fake_logits[:, 1],
92 + 1e-4)
93 +
94 + del model_runner
vllm/lora/layers.py
+12 -8
@@ -10,7 +10,6 @@ from transformers import PretrainedConfig
10
11 from vllm.config import LoRAConfig
12 from vllm.lora.punica import add_lora, add_lora_slice, bgmv
13 -from vllm.model_executor.layers.sampler import Sampler
13 from vllm.model_executor.parallel_utils.communication_op import (
14 tensor_model_parallel_all_gather,
15 tensor_model_parallel_all_reduce,
@@ -20,6 +19,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 RowParallelLinear,
20 QKVParallelLinear,
21 MergedColumnParallelLinear)
22 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 VocabParallelEmbedding, ParallelLMHead)
25 from vllm.model_executor.parallel_utils.parallel_state import (
@@ -783,11 +783,11 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
783 return self.base_layer.weight
784
785
786 -class SamplerWithLoRA(BaseLayerWithLoRA):
786 +class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
787
788 def __init__(
789 self,
790 - base_layer: Sampler,
790 + base_layer: LogitsProcessor,
791 hidden_size: int,
792 dtype: torch.dtype,
793 device: torch.device,
@@ -806,6 +806,10 @@ class SamplerWithLoRA(BaseLayerWithLoRA):
806 def vocab_size(self):
807 return self.base_layer.vocab_size
808
809 + @property
810 + def scale(self):
811 + return self.base_layer.scale
812 +
813 @property
814 def org_vocab_size(self):
815 return self.base_layer.org_vocab_size
@@ -968,14 +972,14 @@ def from_layer(
972 return layer
973
974
971 -def from_layer_sampler(
972 - layer: Sampler,
975 +def from_layer_logits_processor(
976 + layer: LogitsProcessor,
977 lm_head: ParallelLMHead,
978 max_loras: int,
979 lora_config: LoRAConfig,
980 model_config: Optional[PretrainedConfig] = None,
977 -) -> SamplerWithLoRA:
978 - ret = SamplerWithLoRA(layer, lm_head.embedding_dim, lm_head.weight.dtype,
979 - lm_head.weight.device)
981 +) -> LogitsProcessorWithLoRA:
982 + ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
983 + lm_head.weight.dtype, lm_head.weight.device)
984 ret.create_lora_weights(max_loras, lora_config, model_config)
985 return ret
vllm/lora/models.py
+8 -5
@@ -14,7 +14,7 @@ from vllm.config import LoRAConfig
14 from vllm.utils import LRUCache, in_wsl
15
16 from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
17 - from_layer_sampler)
17 + from_layer_logits_processor)
18 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19 from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
20
@@ -421,11 +421,14 @@ class LoRAModelManager:
421 self.model.config))
422 # (yard1): TODO make this more robust
423 if "lm_head" in module_name:
424 - sampler_module = self.model.get_submodule("sampler")
424 + logits_processor_module = self.model.get_submodule(
425 + "logits_processor")
426 new_module = replace_submodule(
426 - self.model, "sampler",
427 - from_layer_sampler(sampler_module, module, self.lora_slots,
428 - self.lora_config, self.model.config))
427 + self.model, "logits_processor",
428 + from_layer_logits_processor(logits_processor_module,
429 + module, self.lora_slots,
430 + self.lora_config,
431 + self.model.config))
432 self.register_module(module_name, new_module)
433 self._register_packed_modules(module_name)
434 new_module.set_mapping(self.base_indices, self.sampler_indices,
vllm/model_executor/layers/logits_processor.py new
+106
@@ -0,0 +1,106 @@
1 +"""A layer that compute logits from hidden_stats."""
2 +from typing import Optional
3 +
4 +import torch
5 +import torch.nn as nn
6 +
7 +from vllm.utils import is_neuron
8 +
9 +from vllm.model_executor.parallel_utils.communication_op import (
10 + tensor_model_parallel_gather)
11 +from vllm.model_executor.sampling_metadata import SamplingMetadata
12 +
13 +
14 +class LogitsProcessor(nn.Module):
15 + """Process logits and apply logits processors from sampling metadata.
16 +
17 + This layer does the following:
18 + 1. Gather logits from model hidden_states.
19 + 2. Scale logits if needed.
20 + 3. Apply logits processors (if any).
21 + """
22 +
23 + def __init__(self,
24 + vocab_size: int,
25 + org_vocab_size: Optional[int] = None,
26 + scale: Optional[float] = 1.0) -> None:
27 + """
28 + Args:
29 + scale: A scaling factor to apply to the logits.
30 + """
31 + super().__init__()
32 + self.scale = scale
33 + self.vocab_size = vocab_size
34 + # Transformers-neuronx generate outputs as logits directly.
35 + self.logits_as_hidden_states = is_neuron()
36 + # original vocabulary size (without LoRA).
37 + self.org_vocab_size = org_vocab_size or vocab_size
38 +
39 + def forward(
40 + self,
41 + embedding: torch.Tensor,
42 + hidden_states: torch.Tensor,
43 + sampling_metadata: SamplingMetadata,
44 + embedding_bias: Optional[torch.Tensor] = None,
45 + ) -> torch.Tensor:
46 + if self.logits_as_hidden_states:
47 + logits = hidden_states
48 + else:
49 + hidden_states = _prune_hidden_states(hidden_states,
50 + sampling_metadata)
51 +
52 + # Get the logits for the next tokens.
53 + logits = self._get_logits(hidden_states, embedding, embedding_bias)
54 +
55 + if logits is not None:
56 + logits *= self.scale
57 +
58 + # Apply logits processors (if any).
59 + logits = _apply_logits_processors(logits, sampling_metadata)
60 +
61 + return logits
62 +
63 + def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
64 + embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
65 + # Get the logits for the next tokens.
66 + logits = torch.matmul(hidden_states, embedding.t())
67 + if embedding_bias is not None:
68 + logits += embedding_bias
69 + logits = tensor_model_parallel_gather(logits)
70 + # Remove paddings in vocab (if any).
71 + if logits is not None:
72 + logits = logits[:, :self.org_vocab_size]
73 + return logits
74 +
75 +
76 +def _prune_hidden_states(
77 + hidden_states: torch.Tensor,
78 + sampling_metadata: SamplingMetadata,
79 +) -> torch.Tensor:
80 + hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
81 + return hidden_states.index_select(0,
82 + sampling_metadata.selected_token_indices)
83 +
84 +
85 +def _apply_logits_processors(
86 + logits: torch.Tensor,
87 + sampling_metadata: SamplingMetadata,
88 +) -> torch.Tensor:
89 + logits_row_idx = 0
90 + found_logits_processors = False
91 + for seq_ids, sampling_params in sampling_metadata.seq_groups:
92 + logits_processors = sampling_params.logits_processors
93 + if logits_processors:
94 + found_logits_processors = True
95 + for seq_id in seq_ids:
96 + logits_row = logits[logits_row_idx]
97 + token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
98 + for logits_processor in logits_processors:
99 + logits_row = logits_processor(token_ids, logits_row)
100 + logits[logits_row_idx] = logits_row
101 + logits_row_idx += 1
102 + else:
103 + logits_row_idx += len(seq_ids)
104 + if found_logits_processors:
105 + assert logits_row_idx == logits.shape[0]
106 + return logits
vllm/model_executor/layers/sampler.py
+1 -80
@@ -4,8 +4,6 @@ from typing import Dict, List, Optional, Tuple
4 import torch
5 import torch.nn as nn
6
7 -from vllm.model_executor.parallel_utils.communication_op import (
8 - tensor_model_parallel_gather)
7 from vllm.model_executor.sampling_metadata import (SamplingMetadata,
8 SamplingTensors)
9 from vllm.sampling_params import SamplingParams, SamplingType
@@ -13,7 +11,6 @@ from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
11 SamplerOutput, SequenceData, SequenceGroupOutput,
12 SequenceOutput)
13 from vllm.model_executor.layers.ops.sample import (sample as sample_triton)
16 -from vllm.utils import is_neuron
14
15
16 class Sampler(nn.Module):
@@ -31,58 +28,14 @@ class Sampler(nn.Module):
28 parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
29 """
30
34 - def __init__(self,
35 - vocab_size: int,
36 - org_vocab_size: Optional[int] = None) -> None:
37 - super().__init__()
38 - self.vocab_size = vocab_size
39 - # Transformers-neuronx generate outputs as logits directly.
40 - self.logits_as_hidden_states = is_neuron()
41 - # original vocabulary size (without LoRA).
42 - self.org_vocab_size = org_vocab_size or vocab_size
43 -
44 - def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
45 - embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
46 - # Get the logits for the next tokens.
47 - logits = torch.matmul(hidden_states, embedding.t())
48 - if embedding_bias is not None:
49 - logits += embedding_bias
50 - logits = tensor_model_parallel_gather(logits)
51 - # Remove paddings in vocab (if any).
52 - if logits is not None:
53 - logits = logits[:, :self.org_vocab_size]
54 - return logits
55 -
31 def forward(
32 self,
58 - embedding: torch.Tensor,
59 - hidden_states: torch.Tensor,
33 + logits: torch.Tensor,
34 sampling_metadata: SamplingMetadata,
61 - embedding_bias: Optional[torch.Tensor] = None,
35 ) -> Optional[SamplerOutput]:
63 - # Get the hidden states that we use for sampling.
64 - if self.logits_as_hidden_states:
65 - logits = hidden_states
66 - else:
67 - hidden_states = _prune_hidden_states(hidden_states,
68 - sampling_metadata)
69 -
70 - # Get the logits for the next tokens.
71 - logits = self._get_logits(hidden_states, embedding, embedding_bias)
72 -
73 - # Only perform sampling in the driver worker.
74 - # Note: `_get_logits` is still distributed across TP workers because
75 - # the `embedding` weight is distributed across TP workers.
76 - # TODO(zhuohan): Change the get_logits part to a separate stage.
77 - if not sampling_metadata.perform_sampling:
78 - return None
79 -
36 assert logits is not None
37 _, vocab_size = logits.shape
38
83 - # Apply logits processors (if any).
84 - logits = _apply_logits_processors(logits, sampling_metadata)
85 -
39 # Prepare sampling tensors with pinned memory to avoid blocking.
40 (sampling_tensors, do_penalties, do_top_p_top_k,
41 do_min_p) = SamplingTensors.from_sampling_metadata(
@@ -124,14 +77,6 @@ class Sampler(nn.Module):
77 prompt_logprobs, sample_logprobs)
78
79
127 -def _prune_hidden_states(
128 - hidden_states: torch.Tensor,
129 - sampling_metadata: SamplingMetadata,
130 -) -> torch.Tensor:
131 - return hidden_states.index_select(0,
132 - sampling_metadata.selected_token_indices)
133 -
134 -
80 def _get_bin_counts_and_mask(
81 tokens: torch.Tensor,
82 vocab_size: int,
@@ -149,30 +94,6 @@ def _get_bin_counts_and_mask(
94 return bin_counts, mask
95
96
152 -def _apply_logits_processors(
153 - logits: torch.Tensor,
154 - sampling_metadata: SamplingMetadata,
155 -) -> torch.Tensor:
156 - logits_row_idx = 0
157 - found_logits_processors = False
158 - for seq_ids, sampling_params in sampling_metadata.seq_groups:
159 - logits_processors = sampling_params.logits_processors
160 - if logits_processors:
161 - found_logits_processors = True
162 - for seq_id in seq_ids:
163 - logits_row = logits[logits_row_idx]
164 - token_ids = sampling_metadata.seq_data[seq_id].output_token_ids
165 - for logits_processor in logits_processors:
166 - logits_row = logits_processor(token_ids, logits_row)
167 - logits[logits_row_idx] = logits_row
168 - logits_row_idx += 1
169 - else:
170 - logits_row_idx += len(seq_ids)
171 - if found_logits_processors:
172 - assert logits_row_idx == logits.shape[0]
173 - return logits
174 -
175 -
97 def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
98 output_tokens_tensor: torch.Tensor,
99 presence_penalties: torch.Tensor,
vllm/model_executor/models/baichuan.py
+11 -4
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
34 QKVParallelLinear,
35 RowParallelLinear)
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.sampler import Sampler
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 VocabParallelEmbedding, ParallelLMHead)
@@ -295,7 +296,8 @@ class BaiChuanBaseForCausalLM(nn.Module):
296 self.linear_method = linear_method
297 self.model = BaiChuanModel(config, position_embedding, linear_method)
298 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
298 - self.sampler = Sampler(config.vocab_size)
299 + self.logits_processor = LogitsProcessor(config.vocab_size)
300 + self.sampler = Sampler()
301
302 def forward(
303 self,
@@ -308,13 +310,18 @@ class BaiChuanBaseForCausalLM(nn.Module):
310 input_metadata)
311 return hidden_states
312
313 + def compute_logits(self, hidden_states: torch.Tensor,
314 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
315 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
316 + sampling_metadata)
317 + return logits
318 +
319 def sample(
320 self,
313 - hidden_states: torch.Tensor,
321 + logits: torch.Tensor,
322 sampling_metadata: SamplingMetadata,
323 ) -> Optional[SamplerOutput]:
316 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
317 - sampling_metadata)
324 + next_tokens = self.sampler(logits, sampling_metadata)
325 return next_tokens
326
327 def load_weights(self,
vllm/model_executor/models/bloom.py
+11 -4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
32 RowParallelLinear)
33 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding)
@@ -273,7 +274,8 @@ class BloomForCausalLM(nn.Module):
274 self.linear_method = linear_method
275 self.transformer = BloomModel(config, linear_method)
276 self.lm_head_weight = self.transformer.word_embeddings.weight
276 - self.sampler = Sampler(config.vocab_size)
277 + self.logits_processor = LogitsProcessor(config.vocab_size)
278 + self.sampler = Sampler()
279
280 def forward(
281 self,
@@ -286,13 +288,18 @@ class BloomForCausalLM(nn.Module):
288 input_metadata)
289 return hidden_states
290
291 + def compute_logits(self, hidden_states: torch.Tensor,
292 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
293 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
294 + sampling_metadata)
295 + return logits
296 +
297 def sample(
298 self,
291 - hidden_states: torch.Tensor,
299 + logits: torch.Tensor,
300 sampling_metadata: SamplingMetadata,
301 ) -> Optional[SamplerOutput]:
294 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
295 - sampling_metadata)
302 + next_tokens = self.sampler(logits, sampling_metadata)
303 return next_tokens
304
305 def load_weights(self,
vllm/model_executor/models/chatglm.py
+11 -4
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
17 QKVParallelLinear,
18 RowParallelLinear)
19 from vllm.model_executor.layers.rotary_embedding import get_rope
20 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 from vllm.model_executor.layers.sampler import Sampler
22 from vllm.model_executor.layers.vocab_parallel_embedding import (
23 VocabParallelEmbedding, ParallelLMHead)
@@ -332,7 +333,8 @@ class ChatGLMForCausalLM(nn.Module):
333 self.linear_method = linear_method
334 self.transformer = ChatGLMModel(config, linear_method)
335 self.lm_head_weight = self.transformer.output_layer.weight
335 - self.sampler = Sampler(config.padded_vocab_size)
336 + self.logits_processor = LogitsProcessor(config.padded_vocab_size)
337 + self.sampler = Sampler()
338
339 def forward(
340 self,
@@ -345,13 +347,18 @@ class ChatGLMForCausalLM(nn.Module):
347 input_metadata)
348 return hidden_states
349
350 + def compute_logits(self, hidden_states: torch.Tensor,
351 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
352 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
353 + sampling_metadata)
354 + return logits
355 +
356 def sample(
357 self,
350 - hidden_states: torch.Tensor,
358 + logits: torch.Tensor,
359 sampling_metadata: SamplingMetadata,
360 ) -> Optional[SamplerOutput]:
353 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
354 - sampling_metadata)
361 + next_tokens = self.sampler(logits, sampling_metadata)
362 return next_tokens
363
364 def load_weights(self,
vllm/model_executor/models/deepseek.py
+11 -4
@@ -38,6 +38,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
38 QKVParallelLinear,
39 RowParallelLinear)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 VocabParallelEmbedding, ParallelLMHead)
@@ -372,7 +373,8 @@ class DeepseekForCausalLM(nn.Module):
373 self.linear_method = linear_method
374 self.model = DeepseekModel(config, linear_method)
375 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
375 - self.sampler = Sampler(config.vocab_size)
376 + self.logits_processor = LogitsProcessor(config.vocab_size)
377 + self.sampler = Sampler()
378
379 def forward(
380 self,
@@ -385,13 +387,18 @@ class DeepseekForCausalLM(nn.Module):
387 input_metadata)
388 return hidden_states
389
390 + def compute_logits(self, hidden_states: torch.Tensor,
391 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
392 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
393 + sampling_metadata)
394 + return logits
395 +
396 def sample(
397 self,
390 - hidden_states: Optional[torch.Tensor],
398 + logits: Optional[torch.Tensor],
399 sampling_metadata: SamplingMetadata,
400 ) -> Optional[SamplerOutput]:
393 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
394 - sampling_metadata)
401 + next_tokens = self.sampler(logits, sampling_metadata)
402 return next_tokens
403
404 def load_weights(self,
vllm/model_executor/models/falcon.py
+11 -4
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 QKVParallelLinear,
35 RowParallelLinear)
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.sampler import Sampler
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 VocabParallelEmbedding, ParallelLMHead)
@@ -373,7 +374,8 @@ class FalconForCausalLM(nn.Module):
374 config.vocab_size,
375 config.hidden_size,
376 )
376 - self.sampler = Sampler(config.vocab_size)
377 + self.logits_processor = LogitsProcessor(config.vocab_size)
378 + self.sampler = Sampler()
379
380 def forward(
381 self,
@@ -390,13 +392,18 @@ class FalconForCausalLM(nn.Module):
392 )
393 return hidden_states
394
395 + def compute_logits(self, hidden_states: torch.Tensor,
396 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
397 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
398 + sampling_metadata)
399 + return logits
400 +
401 def sample(
402 self,
395 - hidden_states: torch.Tensor,
403 + logits: torch.Tensor,
404 sampling_metadata: SamplingMetadata,
405 ) -> Optional[SamplerOutput]:
398 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
399 - sampling_metadata)
406 + next_tokens = self.sampler(logits, sampling_metadata)
407 return next_tokens
408
409 def load_weights(self,
vllm/model_executor/models/gemma.py
+11 -4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
30 QKVParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.rotary_embedding import get_rope
33 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding)
@@ -281,7 +282,8 @@ class GemmaForCausalLM(nn.Module):
282 self.config = config
283 self.linear_method = linear_method
284 self.model = GemmaModel(config, linear_method)
284 - self.sampler = Sampler(config.vocab_size)
285 + self.logits_processor = LogitsProcessor(config.vocab_size)
286 + self.sampler = Sampler()
287
288 @torch.no_grad()
289 def forward(
@@ -295,13 +297,18 @@ class GemmaForCausalLM(nn.Module):
297 input_metadata)
298 return hidden_states
299
300 + def compute_logits(self, hidden_states: torch.Tensor,
301 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
302 + logits = self.logits_processor(self.model.embed_tokens.weight,
303 + hidden_states, sampling_metadata)
304 + return logits
305 +
306 def sample(
307 self,
300 - hidden_states: torch.Tensor,
308 + logits: torch.Tensor,
309 sampling_metadata: SamplingMetadata,
310 ) -> Optional[SamplerOutput]:
303 - next_tokens = self.sampler(self.model.embed_tokens.weight,
304 - hidden_states, sampling_metadata)
311 + next_tokens = self.sampler(logits, sampling_metadata)
312 return next_tokens
313
314 def load_weights(self,
vllm/model_executor/models/gpt2.py
+11 -3
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
32 RowParallelLinear)
33 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding)
@@ -216,7 +217,8 @@ class GPT2LMHeadModel(nn.Module):
217 self.linear_method = linear_method
218 self.transformer = GPT2Model(config, linear_method)
219 self.lm_head_weight = self.transformer.wte.weight
219 - self.sampler = Sampler(config.vocab_size)
220 + self.logits_processor = LogitsProcessor(config.vocab_size)
221 + self.sampler = Sampler()
222
223 def forward(
224 self,
@@ -229,12 +231,18 @@ class GPT2LMHeadModel(nn.Module):
231 input_metadata)
232 return hidden_states
233
234 + def compute_logits(self, hidden_states: torch.Tensor,
235 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
236 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
237 + sampling_metadata)
238 + return logits
239 +
240 def sample(
241 self,
234 - hidden_states: torch.Tensor,
242 + logits: torch.Tensor,
243 sampling_metadata: SamplingMetadata,
244 ) -> Optional[SamplerOutput]:
237 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
245 + next_tokens = self.sampler(self.lm_head_weight, logits,
246 sampling_metadata)
247 return next_tokens
248
vllm/model_executor/models/gpt_bigcode.py
+11 -4
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 LinearMethodBase,
32 QKVParallelLinear,
33 RowParallelLinear)
34 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
@@ -237,7 +238,8 @@ class GPTBigCodeForCausalLM(nn.Module):
238 self.linear_method = linear_method
239 self.transformer = GPTBigCodeModel(config, linear_method)
240 self.lm_head_weight = self.transformer.wte.weight
240 - self.sampler = Sampler(config.vocab_size)
241 + self.logits_processor = LogitsProcessor(config.vocab_size)
242 + self.sampler = Sampler()
243
244 def forward(
245 self,
@@ -250,13 +252,18 @@ class GPTBigCodeForCausalLM(nn.Module):
252 input_metadata)
253 return hidden_states
254
255 + def compute_logits(self, hidden_states: torch.Tensor,
256 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
257 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
258 + sampling_metadata)
259 + return logits
260 +
261 def sample(
262 self,
255 - hidden_states: torch.Tensor,
263 + logits: torch.Tensor,
264 sampling_metadata: SamplingMetadata,
265 ) -> Optional[SamplerOutput]:
258 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
259 - sampling_metadata)
266 + next_tokens = self.sampler(logits, sampling_metadata)
267 return next_tokens
268
269 def load_weights(self,
vllm/model_executor/models/gpt_j.py
+11 -4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 QKVParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.rotary_embedding import get_rope
33 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding, ParallelLMHead)
@@ -224,7 +225,8 @@ class GPTJForCausalLM(nn.Module):
225 config.n_embd,
226 bias=True,
227 )
227 - self.sampler = Sampler(config.vocab_size)
228 + self.logits_processor = LogitsProcessor(config.vocab_size)
229 + self.sampler = Sampler()
230
231 def forward(
232 self,
@@ -237,13 +239,18 @@ class GPTJForCausalLM(nn.Module):
239 input_metadata)
240 return hidden_states
241
242 + def compute_logits(self, hidden_states: torch.Tensor,
243 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
244 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
245 + sampling_metadata, self.lm_head.bias)
246 + return logits
247 +
248 def sample(
249 self,
242 - hidden_states: torch.Tensor,
250 + logits: torch.Tensor,
251 sampling_metadata: SamplingMetadata,
252 ) -> Optional[SamplerOutput]:
245 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
246 - sampling_metadata, self.lm_head.bias)
253 + next_tokens = self.sampler(logits, sampling_metadata)
254 return next_tokens
255
256 def load_weights(self,
vllm/model_executor/models/gpt_neox.py
+11 -4
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 QKVParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.rotary_embedding import get_rope
33 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding, ParallelLMHead)
@@ -238,7 +239,8 @@ class GPTNeoXForCausalLM(nn.Module):
239 config.vocab_size,
240 config.hidden_size,
241 )
241 - self.sampler = Sampler(config.vocab_size)
242 + self.logits_processor = LogitsProcessor(config.vocab_size)
243 + self.sampler = Sampler()
244
245 def forward(
246 self,
@@ -251,13 +253,18 @@ class GPTNeoXForCausalLM(nn.Module):
253 input_metadata)
254 return hidden_states
255
256 + def compute_logits(self, hidden_states: torch.Tensor,
257 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
258 + logits = self.logits_processor(self.embed_out.weight, hidden_states,
259 + sampling_metadata)
260 + return logits
261 +
262 def sample(
263 self,
256 - hidden_states: torch.Tensor,
264 + logits: torch.Tensor,
265 sampling_metadata: SamplingMetadata,
266 ) -> Optional[SamplerOutput]:
259 - next_tokens = self.sampler(self.embed_out.weight, hidden_states,
260 - sampling_metadata)
267 + next_tokens = self.sampler(logits, sampling_metadata)
268 return next_tokens
269
270 def load_weights(self,
vllm/model_executor/models/internlm2.py
+11 -4
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
14 QKVParallelLinear,
15 RowParallelLinear)
16 from vllm.model_executor.layers.rotary_embedding import get_rope
17 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 VocabParallelEmbedding, ParallelLMHead)
@@ -250,7 +251,8 @@ class InternLM2ForCausalLM(nn.Module):
251 self.linear_method = linear_method
252 self.model = InternLM2Model(config, linear_method)
253 self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253 - self.sampler = Sampler(config.vocab_size)
254 + self.logits_processor = LogitsProcessor(config.vocab_size)
255 + self.sampler = Sampler()
256
257 def forward(
258 self,
@@ -263,13 +265,18 @@ class InternLM2ForCausalLM(nn.Module):
265 input_metadata)
266 return hidden_states
267
268 + def compute_logits(self, hidden_states: torch.Tensor,
269 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
270 + logits = self.logits_processor(self.output.weight, hidden_states,
271 + sampling_metadata)
272 + return logits
273 +
274 def sample(
275 self,
268 - hidden_states: torch.Tensor,
276 + logits: torch.Tensor,
277 sampling_metadata: SamplingMetadata,
278 ) -> Optional[SamplerOutput]:
271 - next_tokens = self.sampler(self.output.weight, hidden_states,
272 - sampling_metadata)
279 + next_tokens = self.sampler(logits, sampling_metadata)
280 return next_tokens
281
282 def load_weights(self,
vllm/model_executor/models/llama.py
+14 -4
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
37 QKVParallelLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
@@ -325,7 +326,11 @@ class LlamaForCausalLM(nn.Module):
326 # compatibility
327 if not lora_config else lora_config.lora_vocab_padding_size,
328 )
328 - self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
329 +
330 + logit_scale = getattr(config, "logit_scale", 1.0)
331 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
332 + config.vocab_size, logit_scale)
333 + self.sampler = Sampler()
334
335 def forward(
336 self,
@@ -338,13 +343,18 @@ class LlamaForCausalLM(nn.Module):
343 input_metadata)
344 return hidden_states
345
346 + def compute_logits(self, hidden_states: torch.Tensor,
347 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
348 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
349 + sampling_metadata)
350 + return logits
351 +
352 def sample(
353 self,
343 - hidden_states: torch.Tensor,
354 + logits: torch.Tensor,
355 sampling_metadata: SamplingMetadata,
356 ) -> Optional[SamplerOutput]:
346 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
347 - sampling_metadata)
357 + next_tokens = self.sampler(logits, sampling_metadata)
358 return next_tokens
359
360 def load_weights(self,
vllm/model_executor/models/mixtral.py
+12 -4
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
37 ReplicatedLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
@@ -369,7 +370,9 @@ class MixtralForCausalLM(nn.Module):
370 # compatibility
371 if not lora_config else lora_config.lora_vocab_padding_size,
372 )
372 - self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
373 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
374 + config.vocab_size)
375 + self.sampler = Sampler()
376
377 def forward(
378 self,
@@ -382,13 +385,18 @@ class MixtralForCausalLM(nn.Module):
385 input_metadata)
386 return hidden_states
387
388 + def compute_logits(self, hidden_states: torch.Tensor,
389 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
390 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
391 + sampling_metadata)
392 + return logits
393 +
394 def sample(
395 self,
387 - hidden_states: Optional[torch.Tensor],
396 + logits: Optional[torch.Tensor],
397 sampling_metadata: SamplingMetadata,
398 ) -> Optional[SamplerOutput]:
390 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
391 - sampling_metadata)
399 + next_tokens = self.sampler(logits, sampling_metadata)
400 return next_tokens
401
402 def load_weights(self,
vllm/model_executor/models/mixtral_quant.py
+11 -4
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
39 QKVParallelLinear,
40 RowParallelLinear)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 VocabParallelEmbedding, ParallelLMHead)
@@ -344,7 +345,8 @@ class MixtralForCausalLM(nn.Module):
345 self.linear_method = linear_method
346 self.model = MixtralModel(config, linear_method)
347 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
347 - self.sampler = Sampler(config.vocab_size)
348 + self.logits_processor = LogitsProcessor(config.vocab_size)
349 + self.sampler = Sampler()
350
351 def forward(
352 self,
@@ -357,13 +359,18 @@ class MixtralForCausalLM(nn.Module):
359 input_metadata)
360 return hidden_states
361
362 + def compute_logits(self, hidden_states: torch.Tensor,
363 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
364 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
365 + sampling_metadata)
366 + return logits
367 +
368 def sample(
369 self,
362 - hidden_states: Optional[torch.Tensor],
370 + logits: Optional[torch.Tensor],
371 sampling_metadata: SamplingMetadata,
372 ) -> Optional[SamplerOutput]:
365 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
366 - sampling_metadata)
373 + next_tokens = self.sampler(logits, sampling_metadata)
374 return next_tokens
375
376 def load_weights(self,
vllm/model_executor/models/mpt.py
+11 -4
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13 LinearMethodBase,
14 QKVParallelLinear,
15 RowParallelLinear)
16 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 from vllm.model_executor.layers.sampler import Sampler
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
19 VocabParallelEmbedding)
@@ -259,7 +260,8 @@ class MPTForCausalLM(nn.Module):
260
261 self.transformer = MPTModel(config, linear_method)
262 self.lm_head_weight = self.transformer.wte.weight
262 - self.sampler = Sampler(config.vocab_size)
263 + self.logits_processor = LogitsProcessor(config.vocab_size)
264 + self.sampler = Sampler()
265
266 def forward(
267 self,
@@ -272,13 +274,18 @@ class MPTForCausalLM(nn.Module):
274 input_metadata)
275 return hidden_states
276
277 + def compute_logits(self, hidden_states: torch.Tensor,
278 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
279 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
280 + sampling_metadata)
281 + return logits
282 +
283 def sample(
284 self,
277 - hidden_states: torch.Tensor,
285 + logits: torch.Tensor,
286 sampling_metadata: SamplingMetadata,
287 ) -> Optional[SamplerOutput]:
280 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
281 - sampling_metadata)
288 + next_tokens = self.sampler(logits, sampling_metadata)
289 return next_tokens
290
291 def load_weights(self,
vllm/model_executor/models/neuron/llama.py
+11 -4
@@ -7,6 +7,7 @@ from torch import nn
7 from transformers import LlamaConfig
8
9 from vllm.model_executor.input_metadata import InputMetadata
10 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
11 from vllm.model_executor.layers.sampler import Sampler
12 from vllm.model_executor.sampling_metadata import SamplingMetadata
13 from vllm.sequence import SamplerOutput
@@ -25,7 +26,8 @@ class LlamaForCausalLM(nn.Module):
26 self.config = config
27 self.linear_method = linear_method
28 self.model = None
28 - self.sampler = Sampler(config.vocab_size)
29 + self.logits_processor = LogitsProcessor(config.vocab_size)
30 + self.sampler = Sampler()
31
32 def forward(
33 self,
@@ -45,13 +47,18 @@ class LlamaForCausalLM(nn.Module):
47 start_ids=seq_ids.flatten())
48 return logits
49
50 + def compute_logits(self, hidden_states: torch.Tensor,
51 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
52 + logits = self.logits_processor(self.model.chkpt_model.lm_head,
53 + hidden_states, sampling_metadata)
54 + return logits
55 +
56 def sample(
57 self,
50 - hidden_states: torch.Tensor,
58 + logits: torch.Tensor,
59 sampling_metadata: SamplingMetadata,
60 ) -> Optional[SamplerOutput]:
53 - next_tokens = self.sampler(self.model.chkpt_model.lm_head,
54 - hidden_states, sampling_metadata)
61 + next_tokens = self.sampler(logits, sampling_metadata)
62 return next_tokens
63
64 def load_weights(self,
vllm/model_executor/models/neuron/mistral.py
+11 -4
@@ -6,6 +6,7 @@ from torch import nn
6 from transformers import MistralConfig
7
8 from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
10 from vllm.model_executor.layers.sampler import Sampler
11 from vllm.model_executor.sampling_metadata import SamplingMetadata
12 from vllm.sequence import SamplerOutput
@@ -26,7 +27,8 @@ class MistralForCausalLM(nn.Module):
27 self.linear_method = linear_method
28 self.model = None
29 self.lm_head = None
29 - self.sampler = Sampler(config.vocab_size)
30 + self.logits_processor = LogitsProcessor(config.vocab_size)
31 + self.sampler = Sampler()
32
33 def forward(
34 self,
@@ -48,13 +50,18 @@ class MistralForCausalLM(nn.Module):
50 start_ids=seq_ids)
51 return logits
52
53 + def compute_logits(self, hidden_states: torch.Tensor,
54 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
55 + logits = self.logits_processor(self.model.chkpt_model.lm_head,
56 + hidden_states, sampling_metadata)
57 + return logits
58 +
59 def sample(
60 self,
53 - hidden_states: torch.Tensor,
61 + logits: torch.Tensor,
62 sampling_metadata: SamplingMetadata,
63 ) -> Optional[SamplerOutput]:
56 - next_tokens = self.sampler(self.model.chkpt_model.lm_head,
57 - hidden_states, sampling_metadata)
64 + next_tokens = self.sampler(logits, sampling_metadata)
65 return next_tokens
66
67 def load_weights(self,
vllm/model_executor/models/olmo.py
+11 -4
@@ -51,6 +51,7 @@ from vllm.model_executor.layers.linear import (
51 RowParallelLinear,
52 )
53 from vllm.model_executor.layers.rotary_embedding import get_rope
54 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
55 from vllm.model_executor.layers.sampler import Sampler
56 from vllm.model_executor.layers.vocab_parallel_embedding import (
57 VocabParallelEmbedding)
@@ -336,7 +337,8 @@ class OLMoForCausalLM(nn.Module):
337 self.lm_head_weight = (self.model.transformer.wte.weight
338 if config.weight_tying else
339 self.model.transformer.ff_out.weight)
339 - self.sampler = Sampler(config.vocab_size)
340 + self.logits_processor = LogitsProcessor(config.vocab_size)
341 + self.sampler = Sampler()
342
343 def forward(
344 self,
@@ -353,13 +355,18 @@ class OLMoForCausalLM(nn.Module):
355 )
356 return hidden_states
357
358 + def compute_logits(self, hidden_states: torch.Tensor,
359 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
360 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
361 + sampling_metadata)
362 + return logits
363 +
364 def sample(
365 self,
358 - hidden_states: torch.Tensor,
366 + logits: torch.Tensor,
367 sampling_metadata: SamplingMetadata,
368 ) -> Optional[SamplerOutput]:
361 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
362 - sampling_metadata)
369 + next_tokens = self.sampler(logits, sampling_metadata)
370 return next_tokens
371
372 def load_weights(
vllm/model_executor/models/opt.py
+11 -4
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 QKVParallelLinear,
32 ReplicatedLinear,
33 RowParallelLinear)
34 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
@@ -292,7 +293,8 @@ class OPTForCausalLM(nn.Module):
293 self.linear_method = linear_method
294 self.model = OPTModel(config, linear_method)
295 self.lm_head_weight = self.model.decoder.embed_tokens.weight
295 - self.sampler = Sampler(config.vocab_size)
296 + self.logits_processor = LogitsProcessor(config.vocab_size)
297 + self.sampler = Sampler()
298
299 def forward(
300 self,
@@ -305,13 +307,18 @@ class OPTForCausalLM(nn.Module):
307 input_metadata)
308 return hidden_states
309
310 + def compute_logits(self, hidden_states: torch.Tensor,
311 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
312 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
313 + sampling_metadata)
314 + return logits
315 +
316 def sample(
317 self,
310 - hidden_states: torch.Tensor,
318 + logits: torch.Tensor,
319 sampling_metadata: SamplingMetadata,
320 ) -> Optional[SamplerOutput]:
313 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
314 - sampling_metadata)
321 + next_tokens = self.sampler(logits, sampling_metadata)
322 return next_tokens
323
324 def load_weights(self,
vllm/model_executor/models/orion.py
+11 -4
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
18 QKVParallelLinear,
19 RowParallelLinear)
20 from vllm.model_executor.layers.rotary_embedding import get_rope
21 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
22 from vllm.model_executor.layers.sampler import Sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 VocabParallelEmbedding, ParallelLMHead)
@@ -256,7 +257,8 @@ class OrionForCausalLM(nn.Module):
257 self.linear_method = linear_method
258 self.model = OrionModel(config, linear_method)
259 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
259 - self.sampler = Sampler(config.vocab_size)
260 + self.logits_processor = LogitsProcessor(config.vocab_size)
261 + self.sampler = Sampler()
262
263 def forward(
264 self,
@@ -269,13 +271,18 @@ class OrionForCausalLM(nn.Module):
271 input_metadata)
272 return hidden_states
273
274 + def compute_logits(self, hidden_states: torch.Tensor,
275 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
276 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
277 + sampling_metadata)
278 + return logits
279 +
280 def sample(
281 self,
274 - hidden_states: torch.Tensor,
282 + logits: torch.Tensor,
283 sampling_metadata: SamplingMetadata,
284 ) -> Optional[SamplerOutput]:
277 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
278 - sampling_metadata)
285 + next_tokens = self.sampler(logits, sampling_metadata)
286 return next_tokens
287
288 def load_weights(self,
vllm/model_executor/models/phi.py
+11 -5
@@ -49,6 +49,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49 QKVParallelLinear,
50 RowParallelLinear)
51 from vllm.model_executor.layers.rotary_embedding import get_rope
52 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
53 from vllm.model_executor.layers.sampler import Sampler
54 from vllm.model_executor.layers.vocab_parallel_embedding import (
55 VocabParallelEmbedding, ParallelLMHead)
@@ -240,7 +241,8 @@ class PhiForCausalLM(nn.Module):
241 self.lm_head = ParallelLMHead(config.vocab_size,
242 config.hidden_size,
243 bias=True)
243 - self.sampler = Sampler(config.vocab_size)
244 + self.logits_processor = LogitsProcessor(config.vocab_size)
245 + self.sampler = Sampler()
246
247 def forward(
248 self,
@@ -254,14 +256,18 @@ class PhiForCausalLM(nn.Module):
256
257 return hidden_states
258
259 + def compute_logits(self, hidden_states: torch.Tensor,
260 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
261 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
262 + sampling_metadata, self.lm_head.bias)
263 + return logits
264 +
265 def sample(
266 self,
259 - hidden_states: torch.Tensor,
267 + logits: torch.Tensor,
268 sampling_metadata: SamplingMetadata,
269 ) -> Optional[SamplerOutput]:
262 - head = self.lm_head
263 - next_tokens = self.sampler(head.weight, hidden_states,
264 - sampling_metadata, head.bias)
270 + next_tokens = self.sampler(logits, sampling_metadata)
271 return next_tokens
272
273 def load_weights(self,
vllm/model_executor/models/qwen.py
+11 -4
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
19 QKVParallelLinear,
20 RowParallelLinear)
21 from vllm.model_executor.layers.rotary_embedding import get_rope
22 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
23 from vllm.model_executor.layers.sampler import Sampler
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 VocabParallelEmbedding, ParallelLMHead)
@@ -230,7 +231,8 @@ class QWenLMHeadModel(nn.Module):
231 self.linear_method = linear_method
232 self.transformer = QWenModel(config, linear_method)
233 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
233 - self.sampler = Sampler(config.vocab_size)
234 + self.logits_processor = LogitsProcessor(config.vocab_size)
235 + self.sampler = Sampler()
236
237 def forward(
238 self,
@@ -243,13 +245,18 @@ class QWenLMHeadModel(nn.Module):
245 input_metadata)
246 return hidden_states
247
248 + def compute_logits(self, hidden_states: torch.Tensor,
249 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
250 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
251 + sampling_metadata)
252 + return logits
253 +
254 def sample(
255 self,
248 - hidden_states: torch.Tensor,
256 + logits: torch.Tensor,
257 sampling_metadata: SamplingMetadata,
258 ) -> Optional[SamplerOutput]:
251 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252 - sampling_metadata)
259 + next_tokens = self.sampler(logits, sampling_metadata)
260 return next_tokens
261
262 def load_weights(self,
vllm/model_executor/models/qwen2.py
+15 -9
@@ -37,6 +37,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
37 QKVParallelLinear,
38 RowParallelLinear)
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 VocabParallelEmbedding, ParallelLMHead)
@@ -300,11 +301,15 @@ class Qwen2ForCausalLM(nn.Module):
301 self.linear_method = linear_method
302 self.model = Qwen2Model(config, linear_method)
303
303 - if not config.tie_word_embeddings:
304 + if config.tie_word_embeddings:
305 + self.lm_head_weight = self.model.embed_tokens.weight
306 + else:
307 self.lm_head = ParallelLMHead(config.vocab_size,
308 config.hidden_size)
309 + self.lm_head_weight = self.lm_head.weight
310
307 - self.sampler = Sampler(config.vocab_size)
311 + self.logits_processor = LogitsProcessor(config.vocab_size)
312 + self.sampler = Sampler()
313
314 def forward(
315 self,
@@ -317,17 +322,18 @@ class Qwen2ForCausalLM(nn.Module):
322 input_metadata)
323 return hidden_states
324
325 + def compute_logits(self, hidden_states: torch.Tensor,
326 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
327 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
328 + sampling_metadata)
329 + return logits
330 +
331 def sample(
332 self,
322 - hidden_states: torch.Tensor,
333 + logits: torch.Tensor,
334 sampling_metadata: SamplingMetadata,
335 ) -> Optional[SamplerOutput]:
325 - if self.config.tie_word_embeddings:
326 - lm_head_weight = self.model.embed_tokens.weight
327 - else:
328 - lm_head_weight = self.lm_head.weight
329 - next_tokens = self.sampler(lm_head_weight, hidden_states,
330 - sampling_metadata)
336 + next_tokens = self.sampler(logits, sampling_metadata)
337 return next_tokens
338
339 def load_weights(self,
vllm/model_executor/models/stablelm.py
+11 -4
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
33 QKVParallelLinear,
34 RowParallelLinear)
35 from vllm.model_executor.layers.rotary_embedding import get_rope
36 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding, ParallelLMHead)
@@ -238,7 +239,8 @@ class StablelmForCausalLM(nn.Module):
239 self.linear_method = linear_method
240 self.model = StableLMEpochModel(config, linear_method)
241 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241 - self.sampler = Sampler(config.vocab_size)
242 + self.logits_processor = LogitsProcessor(config.vocab_size)
243 + self.sampler = Sampler()
244
245 def forward(
246 self,
@@ -251,13 +253,18 @@ class StablelmForCausalLM(nn.Module):
253 input_metadata)
254 return hidden_states
255
256 + def compute_logits(self, hidden_states: torch.Tensor,
257 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
258 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
259 + sampling_metadata)
260 + return logits
261 +
262 def sample(
263 self,
256 - hidden_states: torch.Tensor,
264 + logits: torch.Tensor,
265 sampling_metadata: SamplingMetadata,
266 ) -> Optional[SamplerOutput]:
259 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260 - sampling_metadata)
267 + next_tokens = self.sampler(logits, sampling_metadata)
268 return next_tokens
269
270 def load_weights(self,
vllm/model_executor/models/starcoder2.py
+12 -4
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 LinearMethodBase,
33 QKVParallelLinear,
34 RowParallelLinear)
35 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
@@ -254,7 +255,9 @@ class Starcoder2ForCausalLM(nn.Module):
255 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
256 )
257 self.lm_head_weight = self.lm_head.weight
257 - self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
258 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
259 + config.vocab_size)
260 + self.sampler = Sampler()
261
262 def forward(
263 self,
@@ -267,13 +270,18 @@ class Starcoder2ForCausalLM(nn.Module):
270 input_metadata)
271 return hidden_states
272
273 + def compute_logits(self, hidden_states: torch.Tensor,
274 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
275 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
276 + sampling_metadata)
277 + return logits
278 +
279 def sample(
280 self,
272 - hidden_states: Optional[torch.Tensor],
281 + logits: Optional[torch.Tensor],
282 sampling_metadata: SamplingMetadata,
283 ) -> Optional[SamplerOutput]:
275 - next_tokens = self.sampler(self.lm_head_weight, hidden_states,
276 - sampling_metadata)
284 + next_tokens = self.sampler(logits, sampling_metadata)
285 return next_tokens
286
287 def load_weights(self,
vllm/worker/model_runner.py
+8 -1
@@ -613,9 +613,16 @@ class ModelRunner:
613 input_metadata=input_metadata,
614 )
615
616 + # Compute the logits.
617 + logits = self.model.compute_logits(hidden_states, sampling_metadata)
618 +
619 + # Only perform sampling in the driver worker.
620 + if not sampling_metadata.perform_sampling:
621 + return None
622 +
623 # Sample the next token.
624 output = self.model.sample(
618 - hidden_states=hidden_states,
625 + logits=logits,
626 sampling_metadata=sampling_metadata,
627 )
628 return output