A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[BUGFIX] Fix API server test (#2270)
Zhuohan Li committed
Dec 27, 2023 at 02:37 UTC
66b108d1428eaee089bf2a4cb41b1299d150fd93
1 file changed
+8
-6
tests/async_engine/test_api_server.py
+8
-6
@@ -44,13 +44,14 @@ def test_api_server(api_server):
44
"""
45
with Pool(32) as pool:
46
# Wait until the server is ready
47
- prompts = ["Hello world"] * 1
47
+ prompts = ["warm up"] * 1
48
result = None
49
while not result:
50
try:
51
- for _ in pool.map(_query_server, prompts):
51
+ for r in pool.map(_query_server, prompts):
52
+ result = r
53
break
53
- except Exception:
54
+ except requests.exceptions.ConnectionError:
55
time.sleep(1)
56
57
# Actual tests start here
@@ -63,13 +64,14 @@ def test_api_server(api_server):
64
assert num_aborted_requests == 0
65
66
# Try with 100 prompts
66
- prompts = ["Hello world"] * 100
67
+ prompts = ["test prompt"] * 100
68
for result in pool.map(_query_server, prompts):
69
assert result
70
71
# Cancel requests
72
+ prompts = ["canceled requests"] * 100
73
pool.map_async(_query_server, prompts)
72
- time.sleep(0.01)
74
+ time.sleep(0.001)
75
pool.terminate()
76
pool.join()
77
@@ -81,6 +83,6 @@ def test_api_server(api_server):
83
# check that server still runs after cancellations
84
with Pool(32) as pool:
85
# Try with 100 prompts
84
- prompts = ["Hello world"] * 100
86
+ prompts = ["test prompt after canceled"] * 100
87
for result in pool.map(_query_server, prompts):
88
assert result