@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0faab90eb0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[beam search] add output for manually checking the correctness (#8684)

youkaichao committed Sep 20, 2024 at 19:55 UTC 0faab90eb006c677add65cd4c2d0f740a63e064d
1 file changed +10 -3
tests/samplers/test_beam_search.py
+10 -3
@@ -11,7 +11,7 @@ import pytest
11 # 3. Use the model "huggyllama/llama-7b".
12 MAX_TOKENS = [128]
13 BEAM_WIDTHS = [4]
14 -MODELS = ["facebook/opt-125m"]
14 +MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
15
16
17 @pytest.mark.parametrize("model", MODELS)
@@ -37,8 +37,15 @@ def test_beam_search_single_input(
37 beam_width, max_tokens)
38
39 for i in range(len(example_prompts)):
40 - hf_output_ids, _ = hf_outputs[i]
41 - vllm_output_ids, _ = vllm_outputs[i]
40 + hf_output_ids, hf_output_texts = hf_outputs[i]
41 + vllm_output_ids, vllm_output_texts = vllm_outputs[i]
42 + for i, (hf_text,
43 + vllm_text) in enumerate(zip(hf_output_texts,
44 + vllm_output_texts)):
45 + print(f">>>{i}-th hf output:")
46 + print(hf_text)
47 + print(f">>>{i}-th vllm output:")
48 + print(vllm_text)
49 assert len(hf_output_ids) == len(vllm_output_ids)
50 for j in range(len(hf_output_ids)):
51 assert hf_output_ids[j] == vllm_output_ids[j], (