@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 806949514a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[ci] set timeout for test_oot_registration.py (#7082)

youkaichao committed Aug 2, 2024 at 10:03 UTC 806949514ab07a2d7218645022c22962696adf46
3 files changed +10 -2
tests/entrypoints/openai/test_oot_registration.py
+4
@@ -36,10 +36,12 @@ def test_oot_registration_for_api_server():
36 ctx = torch.multiprocessing.get_context()
37 server = ctx.Process(target=server_function, args=(port, ))
38 server.start()
39 + MAX_SERVER_START_WAIT_S = 60
40 client = OpenAI(
41 base_url=f"http://localhost:{port}/v1",
42 api_key="token-abc123",
43 )
44 + now = time.time()
45 while True:
46 try:
47 completion = client.chat.completions.create(
@@ -57,6 +59,8 @@ def test_oot_registration_for_api_server():
59 except OpenAIError as e:
60 if "Connection error" in str(e):
61 time.sleep(3)
62 + if time.time() - now > MAX_SERVER_START_WAIT_S:
63 + raise RuntimeError("Server did not start in time") from e
64 else:
65 raise e
66 server.kill()
vllm/worker/worker.py
+3 -1
@@ -186,7 +186,9 @@ class Worker(LocalOrDistributedWorkerBase):
186 # GPU did not change their memory usage during the profiling.
187 peak_memory = self.init_gpu_memory - free_gpu_memory
188 assert peak_memory > 0, (
189 - "Error in memory profiling. This happens when the GPU memory was "
189 + "Error in memory profiling. "
190 + f"Initial free memory {self.init_gpu_memory}, current free memory"
191 + f" {free_gpu_memory}. This happens when the GPU memory was "
192 "not properly cleaned up before initializing the vLLM instance.")
193
194 cache_block_size = self.get_cache_block_size_bytes()
vllm/worker/xpu_worker.py
+3 -1
@@ -138,7 +138,9 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
138 # GPU did not change their memory usage during the profiling.
139 peak_memory = self.init_gpu_memory - free_gpu_memory
140 assert peak_memory > 0, (
141 - "Error in memory profiling. This happens when the GPU memory was "
141 + "Error in memory profiling. "
142 + f"Initial free memory {self.init_gpu_memory}, current free memory"
143 + f" {free_gpu_memory}. This happens when the GPU memory was "
144 "not properly cleaned up before initializing the vLLM instance.")
145
146 cache_block_size = self.get_cache_block_size_bytes()