A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Doc]Add documentation to benchmarking script when running TGI (#4920)
Kuntai Du committed
May 20, 2024 at 13:16 UTC
c3af44722cff56bba5fc912c8e16d9de02dfb532
2 files changed
+5
-1
benchmarks/benchmark_serving.py
+4
@@ -17,6 +17,10 @@ On the client side, run:
17
--dataset-path <path to dataset> \
18
--request-rate <request_rate> \ # By default <request_rate> is inf
19
--num-prompts <num_prompts> # By default <num_prompts> is 1000
20
+
21
+ when using tgi backend, add
22
+ --endpoint /generate_stream
23
+ to the end of the command above.
24
"""
25
import argparse
26
import asyncio
benchmarks/launch_tgi_server.sh
+1
-1
@@ -4,7 +4,7 @@ PORT=8000
4
MODEL=$1
5
TOKENS=$2
6
7
-docker run --gpus all --shm-size 1g -p $PORT:80 \
7
+docker run -e HF_TOKEN=$HF_TOKEN --gpus all --shm-size 1g -p $PORT:80 \
8
-v $PWD/data:/data \
9
ghcr.io/huggingface/text-generation-inference:1.4.0 \
10
--model-id $MODEL \