@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66b108d142
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[BUGFIX] Fix API server test (#2270)

Zhuohan Li committed Dec 27, 2023 at 02:37 UTC 66b108d1428eaee089bf2a4cb41b1299d150fd93
1 file changed +8 -6
tests/async_engine/test_api_server.py
+8 -6
@@ -44,13 +44,14 @@ def test_api_server(api_server):
44 """
45 with Pool(32) as pool:
46 # Wait until the server is ready
47 - prompts = ["Hello world"] * 1
47 + prompts = ["warm up"] * 1
48 result = None
49 while not result:
50 try:
51 - for _ in pool.map(_query_server, prompts):
51 + for r in pool.map(_query_server, prompts):
52 + result = r
53 break
53 - except Exception:
54 + except requests.exceptions.ConnectionError:
55 time.sleep(1)
56
57 # Actual tests start here
@@ -63,13 +64,14 @@ def test_api_server(api_server):
64 assert num_aborted_requests == 0
65
66 # Try with 100 prompts
66 - prompts = ["Hello world"] * 100
67 + prompts = ["test prompt"] * 100
68 for result in pool.map(_query_server, prompts):
69 assert result
70
71 # Cancel requests
72 + prompts = ["canceled requests"] * 100
73 pool.map_async(_query_server, prompts)
72 - time.sleep(0.01)
74 + time.sleep(0.001)
75 pool.terminate()
76 pool.join()
77
@@ -81,6 +83,6 @@ def test_api_server(api_server):
83 # check that server still runs after cancellations
84 with Pool(32) as pool:
85 # Try with 100 prompts
84 - prompts = ["Hello world"] * 100
86 + prompts = ["test prompt after canceled"] * 100
87 for result in pool.map(_query_server, prompts):
88 assert result