A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[beam search] add output for manually checking the correctness (#8684)
youkaichao committed
Sep 20, 2024 at 19:55 UTC
0faab90eb006c677add65cd4c2d0f740a63e064d
1 file changed
+10
-3
tests/samplers/test_beam_search.py
+10
-3
@@ -11,7 +11,7 @@ import pytest
11
# 3. Use the model "huggyllama/llama-7b".
12
MAX_TOKENS = [128]
13
BEAM_WIDTHS = [4]
14
-MODELS = ["facebook/opt-125m"]
14
+MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
15
16
17
@pytest.mark.parametrize("model", MODELS)
@@ -37,8 +37,15 @@ def test_beam_search_single_input(
37
beam_width, max_tokens)
38
39
for i in range(len(example_prompts)):
40
- hf_output_ids, _ = hf_outputs[i]
41
- vllm_output_ids, _ = vllm_outputs[i]
40
+ hf_output_ids, hf_output_texts = hf_outputs[i]
41
+ vllm_output_ids, vllm_output_texts = vllm_outputs[i]
42
+ for i, (hf_text,
43
+ vllm_text) in enumerate(zip(hf_output_texts,
44
+ vllm_output_texts)):
45
+ print(f">>>{i}-th hf output:")
46
+ print(hf_text)
47
+ print(f">>>{i}-th vllm output:")
48
+ print(vllm_text)
49
assert len(hf_output_ids) == len(vllm_output_ids)
50
for j in range(len(hf_output_ids)):
51
assert hf_output_ids[j] == vllm_output_ids[j], (