A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Sort the outputs before return (#402)
Woosuk Kwon committed
Jul 8, 2023 at 14:48 UTC
b6fbb9a565a7e27d26e91a064dc83d13b36fa2b6
1 file changed
+4
vllm/entrypoints/llm.py
+4
@@ -155,4 +155,8 @@ class LLM:
155
pbar.update(1)
156
if use_tqdm:
157
pbar.close()
158
+ # Sort the outputs by request ID.
159
+ # This is necessary because some requests may be finished earlier than
160
+ # its previous requests.
161
+ outputs = sorted(outputs, key=lambda x: int(x.request_id))
162
return outputs