A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix][Example] Fix GCed profiling server for TPU (#12792)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Feb 12, 2025 at 22:52 UTC
d88c8666a167c97193cb3d48fd2de3d90b5b8d85
1 file changed
+1
-1
examples/offline_inference/profiling_tpu/profiling.py
+1
-1
@@ -24,7 +24,7 @@ def main(args: argparse.Namespace):
24
25
engine_args = EngineArgs.from_cli_args(args)
26
llm = LLM(**dataclasses.asdict(engine_args))
27
- _ = xp.start_server(9012)
27
+ server = xp.start_server(9012) # noqa: F841
28
29
sampling_params = SamplingParams(
30
temperature=0.0,