@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3a5bd9fb7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix sampler test (#1379)

Woosuk Kwon committed Oct 16, 2023 at 12:57 UTC d3a5bd9fb7d778c2f2f74bcf8d5343f185f69b61
1 file changed +4 -4
tests/samplers/test_sampler.py
+4 -4
@@ -1,9 +1,9 @@
1 # pylint: disable=protected-access
2 -import pytest
2 import random
3 from typing import Tuple
4 from unittest.mock import patch
5
6 +import pytest
7 import torch
8
9 from vllm.model_executor.layers.sampler import Sampler
@@ -69,7 +69,7 @@ def test_sampler_all_greedy(seed: int):
69 input_metadata=input_metadata)
70 expected = torch.argmax(fake_logits, dim=-1)
71 for i, sequence_output in enumerate(sampler_output):
72 - for nth_output in sequence_output:
72 + for nth_output in sequence_output.samples:
73 assert nth_output.output_token == expected[i].item()
74
75
@@ -101,7 +101,7 @@ def test_sampler_all_random(seed: int):
101 hidden_states=input_tensor,
102 input_metadata=input_metadata)
103 for i, sequence_output in enumerate(sampler_output):
104 - for nth_output in sequence_output:
104 + for nth_output in sequence_output.samples:
105 assert nth_output.output_token == i
106
107
@@ -181,5 +181,5 @@ def test_sampler_mixed(seed: int):
181 for i, sequence_output in enumerate(sampler_output):
182 if seq_group_metadata_list[i].sampling_params.use_beam_search:
183 continue
184 - for nth_output in sequence_output:
184 + for nth_output in sequence_output.samples:
185 assert nth_output.output_token in expected_tokens