A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc] use AutoTokenizer for benchmark serving when vLLM not installed (#5588)
zhyncs committed
Jun 18, 2024 at 00:40 UTC
1f12122b1714c855c02699775bcd2fb2b34f2577
2 files changed
+32
-2
benchmarks/backend_request_func.py
+28
-1
@@ -4,10 +4,13 @@ import sys
4
import time
5
import traceback
6
from dataclasses import dataclass, field
7
-from typing import List, Optional
7
+from typing import List, Optional, Union
8
9
import aiohttp
10
+import huggingface_hub.constants
11
from tqdm.asyncio import tqdm
12
+from transformers import (AutoTokenizer, PreTrainedTokenizer,
13
+ PreTrainedTokenizerFast)
14
15
AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
16
@@ -388,6 +391,30 @@ def remove_prefix(text: str, prefix: str) -> str:
391
return text
392
393
394
+def get_model(pretrained_model_name_or_path: str):
395
+ if os.getenv('VLLM_USE_MODELSCOPE', 'False').lower() == 'true':
396
+ from modelscope import snapshot_download
397
+ else:
398
+ from huggingface_hub import snapshot_download
399
+
400
+ model_path = snapshot_download(
401
+ model_id=pretrained_model_name_or_path,
402
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
403
+ ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
404
+ return model_path
405
+
406
+
407
+def get_tokenizer(
408
+ pretrained_model_name_or_path: str, trust_remote_code: bool
409
+) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
410
+ if pretrained_model_name_or_path is not None and not os.path.exists(
411
+ pretrained_model_name_or_path):
412
+ pretrained_model_name_or_path = get_model(
413
+ pretrained_model_name_or_path)
414
+ return AutoTokenizer.from_pretrained(pretrained_model_name_or_path,
415
+ trust_remote_code=trust_remote_code)
416
+
417
+
418
ASYNC_REQUEST_FUNCS = {
419
"tgi": async_request_tgi,
420
"vllm": async_request_openai_completions,
benchmarks/benchmark_serving.py
+4
-1
@@ -39,7 +39,10 @@ from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
39
from tqdm.asyncio import tqdm
40
from transformers import PreTrainedTokenizerBase
41
42
-from vllm.transformers_utils.tokenizer import get_tokenizer
42
+try:
43
+ from vllm.transformers_utils.tokenizer import get_tokenizer
44
+except ImportError:
45
+ from backend_request_func import get_tokenizer
46
47
48
@dataclass