@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cb08cd0d75
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Minor] Fix duplication of ignored seq group in engine step (#1666)

Simon Mo committed Nov 16, 2023 at 13:11 UTC cb08cd0d754f5ac352e31bf362062ba61403eb02
2 files changed +28 -1
tests/test_regression.py new
+27
@@ -0,0 +1,27 @@
1 +"""Containing tests that check for regressions in vLLM's behavior.
2 +
3 +It should include tests that are reported by users and making sure they
4 +will never happen again.
5 +
6 +"""
7 +from vllm import LLM, SamplingParams
8 +
9 +
10 +def test_duplicated_ignored_sequence_group():
11 + """https://github.com/vllm-project/vllm/issues/1655"""
12 +
13 + sampling_params = SamplingParams(temperature=0.01,
14 + top_p=0.1,
15 + max_tokens=256)
16 + llm = LLM(model="facebook/opt-125m",
17 + max_num_batched_tokens=4096,
18 + tensor_parallel_size=1)
19 + prompts = ["This is a short prompt", "This is a very long prompt " * 1000]
20 + outputs = llm.generate(prompts, sampling_params=sampling_params)
21 +
22 + assert len(prompts) == len(outputs)
23 +
24 +
25 +if __name__ == "__main__":
26 + import pytest
27 + pytest.main([__file__])
vllm/engine/llm_engine.py
+1 -1
@@ -567,7 +567,7 @@ class LLMEngine:
567 blocks_to_copy=scheduler_outputs.blocks_to_copy,
568 )
569
570 - return self._process_model_outputs(output, scheduler_outputs) + ignored
570 + return self._process_model_outputs(output, scheduler_outputs)
571
572 def _log_system_stats(
573 self,