A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Misc][Tools][Benchmark] Add benchmark_serving supports for llama.cpp. (#18692)
Signed-off-by: Duyi-Wang <duyi.wang@intel.com>
Duyi-Wang committed
May 29, 2025 at 20:02 UTC
b169d5f7b6ad65586c5352030bda154d589a4d89
2 files changed
+6
-1
benchmarks/backend_request_func.py
+2
-1
@@ -324,7 +324,7 @@ async def async_request_openai_completions(
324
325
most_recent_timestamp = timestamp
326
generated_text += text or ""
327
- elif usage := data.get("usage"):
327
+ if usage := data.get("usage"):
328
output.output_tokens = usage.get("completion_tokens")
329
if first_chunk_received:
330
output.success = True
@@ -611,6 +611,7 @@ ASYNC_REQUEST_FUNCS = {
611
"tensorrt-llm": async_request_trt_llm,
612
"scalellm": async_request_openai_completions,
613
"sglang": async_request_openai_completions,
614
+ "llama.cpp": async_request_openai_completions,
615
}
616
617
OPENAI_COMPATIBLE_BACKENDS = [
benchmarks/benchmark_serving.py
+4
@@ -762,6 +762,10 @@ def main(args: argparse.Namespace):
762
if "temperature" not in sampling_params:
763
sampling_params["temperature"] = 0.0 # Default to greedy decoding.
764
765
+ if args.backend == "llama.cpp":
766
+ # Disable prompt caching in llama.cpp backend
767
+ sampling_params["cache_prompt"] = False
768
+
769
# Avoid GC processing "static" data - reduce pause times.
770
gc.collect()
771
gc.freeze()