A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[ci] set timeout for test_oot_registration.py (#7082)
youkaichao committed
Aug 2, 2024 at 10:03 UTC
806949514ab07a2d7218645022c22962696adf46
3 files changed
+10
-2
tests/entrypoints/openai/test_oot_registration.py
+4
@@ -36,10 +36,12 @@ def test_oot_registration_for_api_server():
36
ctx = torch.multiprocessing.get_context()
37
server = ctx.Process(target=server_function, args=(port, ))
38
server.start()
39
+ MAX_SERVER_START_WAIT_S = 60
40
client = OpenAI(
41
base_url=f"http://localhost:{port}/v1",
42
api_key="token-abc123",
43
)
44
+ now = time.time()
45
while True:
46
try:
47
completion = client.chat.completions.create(
@@ -57,6 +59,8 @@ def test_oot_registration_for_api_server():
59
except OpenAIError as e:
60
if "Connection error" in str(e):
61
time.sleep(3)
62
+ if time.time() - now > MAX_SERVER_START_WAIT_S:
63
+ raise RuntimeError("Server did not start in time") from e
64
else:
65
raise e
66
server.kill()
vllm/worker/worker.py
+3
-1
@@ -186,7 +186,9 @@ class Worker(LocalOrDistributedWorkerBase):
186
# GPU did not change their memory usage during the profiling.
187
peak_memory = self.init_gpu_memory - free_gpu_memory
188
assert peak_memory > 0, (
189
- "Error in memory profiling. This happens when the GPU memory was "
189
+ "Error in memory profiling. "
190
+ f"Initial free memory {self.init_gpu_memory}, current free memory"
191
+ f" {free_gpu_memory}. This happens when the GPU memory was "
192
"not properly cleaned up before initializing the vLLM instance.")
193
194
cache_block_size = self.get_cache_block_size_bytes()
vllm/worker/xpu_worker.py
+3
-1
@@ -138,7 +138,9 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
138
# GPU did not change their memory usage during the profiling.
139
peak_memory = self.init_gpu_memory - free_gpu_memory
140
assert peak_memory > 0, (
141
- "Error in memory profiling. This happens when the GPU memory was "
141
+ "Error in memory profiling. "
142
+ f"Initial free memory {self.init_gpu_memory}, current free memory"
143
+ f" {free_gpu_memory}. This happens when the GPU memory was "
144
"not properly cleaned up before initializing the vLLM instance.")
145
146
cache_block_size = self.get_cache_block_size_bytes()