A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix sampler test (#1379)
Woosuk Kwon committed
Oct 16, 2023 at 12:57 UTC
d3a5bd9fb7d778c2f2f74bcf8d5343f185f69b61
1 file changed
+4
-4
tests/samplers/test_sampler.py
+4
-4
@@ -1,9 +1,9 @@
1
# pylint: disable=protected-access
2
-import pytest
2
import random
3
from typing import Tuple
4
from unittest.mock import patch
5
6
+import pytest
7
import torch
8
9
from vllm.model_executor.layers.sampler import Sampler
@@ -69,7 +69,7 @@ def test_sampler_all_greedy(seed: int):
69
input_metadata=input_metadata)
70
expected = torch.argmax(fake_logits, dim=-1)
71
for i, sequence_output in enumerate(sampler_output):
72
- for nth_output in sequence_output:
72
+ for nth_output in sequence_output.samples:
73
assert nth_output.output_token == expected[i].item()
74
75
@@ -101,7 +101,7 @@ def test_sampler_all_random(seed: int):
101
hidden_states=input_tensor,
102
input_metadata=input_metadata)
103
for i, sequence_output in enumerate(sampler_output):
104
- for nth_output in sequence_output:
104
+ for nth_output in sequence_output.samples:
105
assert nth_output.output_token == i
106
107
@@ -181,5 +181,5 @@ def test_sampler_mixed(seed: int):
181
for i, sequence_output in enumerate(sampler_output):
182
if seq_group_metadata_list[i].sampling_params.use_beam_search:
183
continue
184
- for nth_output in sequence_output:
184
+ for nth_output in sequence_output.samples:
185
assert nth_output.output_token in expected_tokens