A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[HotFix] Fix final output truncation with stop string + streaming (#8468)
Nick Hill committed
Sep 13, 2024 at 19:31 UTC
18e9e1f7b34c46857466fe24e9f9bdee17542f2c
2 files changed
+24
-6
tests/async_engine/test_async_llm_engine.py
+21
-5
@@ -159,7 +159,8 @@ def should_do_global_cleanup_after_test(request) -> bool:
159
160
161
@pytest.mark.asyncio(scope="module")
162
-async def test_asyncio_run(async_engine):
162
+@pytest.mark.parametrize("stop", [None, ["a stop string"]])
163
+async def test_asyncio_run(async_engine, stop):
164
165
scheduler_config = await async_engine.get_scheduler_config()
166
num_scheduler_steps = scheduler_config.num_scheduler_steps
@@ -169,6 +170,7 @@ async def test_asyncio_run(async_engine):
170
temperature=0,
171
max_tokens=32,
172
min_tokens=32,
173
+ stop=stop,
174
)
175
176
output_count = 0
@@ -203,7 +205,8 @@ async def test_asyncio_run(async_engine):
205
206
207
@pytest.mark.asyncio(scope="module")
206
-async def test_output_kinds(async_engine):
208
+@pytest.mark.parametrize("stop", [None, ["a stop string"]])
209
+async def test_output_kinds(async_engine, stop):
210
"""Test that output_kind works as expected and that
211
results are equivalent across different kinds."""
212
@@ -214,6 +217,7 @@ async def test_output_kinds(async_engine):
217
temperature=0,
218
max_tokens=32,
219
min_tokens=32,
220
+ stop=stop,
221
)
222
223
async def run(prompt: str, kind: RequestOutputKind):
@@ -229,6 +233,8 @@ async def test_output_kinds(async_engine):
233
final_output = output
234
235
assert final_output is not None
236
+ assert final_output.finished
237
+
238
return (final_output.prompt_token_ids,
239
final_output.outputs[0].token_ids,
240
final_output.outputs[0].text, output_count)
@@ -241,16 +247,18 @@ async def test_output_kinds(async_engine):
247
output_tokens: List[int] = []
248
output_text = ""
249
output_count = 0
250
+ final_output = None
251
async for output in async_engine.generate(prompt,
252
params,
253
request_id=uid()):
254
token_ids = output.outputs[0].token_ids
255
text = output.outputs[0].text
256
+ final_output = output
257
258
# Ensure we get prompt ids iff we haven't yet received output tokens
259
if output_tokens:
260
assert 1 <= len(token_ids) <= num_scheduler_steps
253
- assert text
261
+ assert stop or text
262
assert not output.prompt_token_ids
263
else:
264
assert output.prompt_token_ids
@@ -260,6 +268,10 @@ async def test_output_kinds(async_engine):
268
output_text += text
269
270
output_count += 1
271
+
272
+ assert final_output is not None
273
+ assert final_output.finished
274
+
275
return prompt_tokens, output_tokens, output_text, output_count
276
277
results = await asyncio.gather(
@@ -291,7 +303,8 @@ async def test_output_kinds(async_engine):
303
304
305
@pytest.mark.asyncio(scope="module")
294
-async def test_cancellation(async_engine):
306
+@pytest.mark.parametrize("stop", [None, ["a stop string"]])
307
+async def test_cancellation(async_engine, stop):
308
scheduler_config = await async_engine.get_scheduler_config()
309
num_scheduler_steps = scheduler_config.num_scheduler_steps
310
@@ -299,6 +312,7 @@ async def test_cancellation(async_engine):
312
temperature=0,
313
min_tokens=13,
314
max_tokens=13,
315
+ stop=stop,
316
)
317
318
stop_at = 5 if num_scheduler_steps == 1 else 1
@@ -319,7 +333,8 @@ async def test_cancellation(async_engine):
333
334
335
@pytest.mark.asyncio(scope="module")
322
-async def test_delayed_generator(async_engine):
336
+@pytest.mark.parametrize("stop", [None, ["a stop string"]])
337
+async def test_delayed_generator(async_engine, stop):
338
scheduler_config = await async_engine.get_scheduler_config()
339
340
if scheduler_config.num_scheduler_steps != 1:
@@ -329,6 +344,7 @@ async def test_delayed_generator(async_engine):
344
temperature=0,
345
min_tokens=10,
346
max_tokens=10,
347
+ stop=stop,
348
)
349
350
stream = async_engine.generate("test3", sampling_params, request_id=uid())
vllm/sequence.py
+3
-1
@@ -477,7 +477,9 @@ class Sequence:
477
if not delta:
478
return self.output_text[:-buffer_length] if truncate else (
479
self.output_text)
480
- length = len(self.output_text) - buffer_length
480
+ length = len(self.output_text)
481
+ if truncate:
482
+ length -= buffer_length
483
last_offset = self._last_output_text_offset
484
if last_offset < length:
485
self._last_output_text_offset = length