@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18e9e1f7b3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[HotFix] Fix final output truncation with stop string + streaming (#8468)

Nick Hill committed Sep 13, 2024 at 19:31 UTC 18e9e1f7b34c46857466fe24e9f9bdee17542f2c
2 files changed +24 -6
tests/async_engine/test_async_llm_engine.py
+21 -5
@@ -159,7 +159,8 @@ def should_do_global_cleanup_after_test(request) -> bool:
159
160
161 @pytest.mark.asyncio(scope="module")
162 -async def test_asyncio_run(async_engine):
162 +@pytest.mark.parametrize("stop", [None, ["a stop string"]])
163 +async def test_asyncio_run(async_engine, stop):
164
165 scheduler_config = await async_engine.get_scheduler_config()
166 num_scheduler_steps = scheduler_config.num_scheduler_steps
@@ -169,6 +170,7 @@ async def test_asyncio_run(async_engine):
170 temperature=0,
171 max_tokens=32,
172 min_tokens=32,
173 + stop=stop,
174 )
175
176 output_count = 0
@@ -203,7 +205,8 @@ async def test_asyncio_run(async_engine):
205
206
207 @pytest.mark.asyncio(scope="module")
206 -async def test_output_kinds(async_engine):
208 +@pytest.mark.parametrize("stop", [None, ["a stop string"]])
209 +async def test_output_kinds(async_engine, stop):
210 """Test that output_kind works as expected and that
211 results are equivalent across different kinds."""
212
@@ -214,6 +217,7 @@ async def test_output_kinds(async_engine):
217 temperature=0,
218 max_tokens=32,
219 min_tokens=32,
220 + stop=stop,
221 )
222
223 async def run(prompt: str, kind: RequestOutputKind):
@@ -229,6 +233,8 @@ async def test_output_kinds(async_engine):
233 final_output = output
234
235 assert final_output is not None
236 + assert final_output.finished
237 +
238 return (final_output.prompt_token_ids,
239 final_output.outputs[0].token_ids,
240 final_output.outputs[0].text, output_count)
@@ -241,16 +247,18 @@ async def test_output_kinds(async_engine):
247 output_tokens: List[int] = []
248 output_text = ""
249 output_count = 0
250 + final_output = None
251 async for output in async_engine.generate(prompt,
252 params,
253 request_id=uid()):
254 token_ids = output.outputs[0].token_ids
255 text = output.outputs[0].text
256 + final_output = output
257
258 # Ensure we get prompt ids iff we haven't yet received output tokens
259 if output_tokens:
260 assert 1 <= len(token_ids) <= num_scheduler_steps
253 - assert text
261 + assert stop or text
262 assert not output.prompt_token_ids
263 else:
264 assert output.prompt_token_ids
@@ -260,6 +268,10 @@ async def test_output_kinds(async_engine):
268 output_text += text
269
270 output_count += 1
271 +
272 + assert final_output is not None
273 + assert final_output.finished
274 +
275 return prompt_tokens, output_tokens, output_text, output_count
276
277 results = await asyncio.gather(
@@ -291,7 +303,8 @@ async def test_output_kinds(async_engine):
303
304
305 @pytest.mark.asyncio(scope="module")
294 -async def test_cancellation(async_engine):
306 +@pytest.mark.parametrize("stop", [None, ["a stop string"]])
307 +async def test_cancellation(async_engine, stop):
308 scheduler_config = await async_engine.get_scheduler_config()
309 num_scheduler_steps = scheduler_config.num_scheduler_steps
310
@@ -299,6 +312,7 @@ async def test_cancellation(async_engine):
312 temperature=0,
313 min_tokens=13,
314 max_tokens=13,
315 + stop=stop,
316 )
317
318 stop_at = 5 if num_scheduler_steps == 1 else 1
@@ -319,7 +333,8 @@ async def test_cancellation(async_engine):
333
334
335 @pytest.mark.asyncio(scope="module")
322 -async def test_delayed_generator(async_engine):
336 +@pytest.mark.parametrize("stop", [None, ["a stop string"]])
337 +async def test_delayed_generator(async_engine, stop):
338 scheduler_config = await async_engine.get_scheduler_config()
339
340 if scheduler_config.num_scheduler_steps != 1:
@@ -329,6 +344,7 @@ async def test_delayed_generator(async_engine):
344 temperature=0,
345 min_tokens=10,
346 max_tokens=10,
347 + stop=stop,
348 )
349
350 stream = async_engine.generate("test3", sampling_params, request_id=uid())
vllm/sequence.py
+3 -1
@@ -477,7 +477,9 @@ class Sequence:
477 if not delta:
478 return self.output_text[:-buffer_length] if truncate else (
479 self.output_text)
480 - length = len(self.output_text) - buffer_length
480 + length = len(self.output_text)
481 + if truncate:
482 + length -= buffer_length
483 last_offset = self._last_output_text_offset
484 if last_offset < length:
485 self._last_output_text_offset = length