A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Minor] Fix duplication of ignored seq group in engine step (#1666)
Simon Mo committed
Nov 16, 2023 at 13:11 UTC
cb08cd0d754f5ac352e31bf362062ba61403eb02
2 files changed
+28
-1
tests/test_regression.py
new
+27
@@ -0,0 +1,27 @@
1
+"""Containing tests that check for regressions in vLLM's behavior.
2
+
3
+It should include tests that are reported by users and making sure they
4
+will never happen again.
5
+
6
+"""
7
+from vllm import LLM, SamplingParams
8
+
9
+
10
+def test_duplicated_ignored_sequence_group():
11
+ """https://github.com/vllm-project/vllm/issues/1655"""
12
+
13
+ sampling_params = SamplingParams(temperature=0.01,
14
+ top_p=0.1,
15
+ max_tokens=256)
16
+ llm = LLM(model="facebook/opt-125m",
17
+ max_num_batched_tokens=4096,
18
+ tensor_parallel_size=1)
19
+ prompts = ["This is a short prompt", "This is a very long prompt " * 1000]
20
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
21
+
22
+ assert len(prompts) == len(outputs)
23
+
24
+
25
+if __name__ == "__main__":
26
+ import pytest
27
+ pytest.main([__file__])
vllm/engine/llm_engine.py
+1
-1
@@ -567,7 +567,7 @@ class LLMEngine:
567
blocks_to_copy=scheduler_outputs.blocks_to_copy,
568
)
569
570
- return self._process_model_outputs(output, scheduler_outputs) + ignored
570
+ return self._process_model_outputs(output, scheduler_outputs)
571
572
def _log_system_stats(
573
self,