A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[benchmark] Update TGI version (#7917)
Philipp Schmid committed
Aug 28, 2024 at 00:07 UTC
345be0e2445f82bb6bed166c205feeb4f4f73fc3
1 file changed
+1
-1
benchmarks/launch_tgi_server.sh
+1
-1
@@ -6,7 +6,7 @@ TOKENS=$2
6
7
docker run -e HF_TOKEN=$HF_TOKEN --gpus all --shm-size 1g -p $PORT:80 \
8
-v $PWD/data:/data \
9
- ghcr.io/huggingface/text-generation-inference:1.4.0 \
9
+ ghcr.io/huggingface/text-generation-inference:2.2.0 \
10
--model-id $MODEL \
11
--sharded false \
12
--max-input-length 1024 \