@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f12122b17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] use AutoTokenizer for benchmark serving when vLLM not installed (#5588)

zhyncs committed Jun 18, 2024 at 00:40 UTC 1f12122b1714c855c02699775bcd2fb2b34f2577
2 files changed +32 -2
benchmarks/backend_request_func.py
+28 -1
@@ -4,10 +4,13 @@ import sys
4 import time
5 import traceback
6 from dataclasses import dataclass, field
7 -from typing import List, Optional
7 +from typing import List, Optional, Union
8
9 import aiohttp
10 +import huggingface_hub.constants
11 from tqdm.asyncio import tqdm
12 +from transformers import (AutoTokenizer, PreTrainedTokenizer,
13 + PreTrainedTokenizerFast)
14
15 AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
16
@@ -388,6 +391,30 @@ def remove_prefix(text: str, prefix: str) -> str:
391 return text
392
393
394 +def get_model(pretrained_model_name_or_path: str):
395 + if os.getenv('VLLM_USE_MODELSCOPE', 'False').lower() == 'true':
396 + from modelscope import snapshot_download
397 + else:
398 + from huggingface_hub import snapshot_download
399 +
400 + model_path = snapshot_download(
401 + model_id=pretrained_model_name_or_path,
402 + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
403 + ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
404 + return model_path
405 +
406 +
407 +def get_tokenizer(
408 + pretrained_model_name_or_path: str, trust_remote_code: bool
409 +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
410 + if pretrained_model_name_or_path is not None and not os.path.exists(
411 + pretrained_model_name_or_path):
412 + pretrained_model_name_or_path = get_model(
413 + pretrained_model_name_or_path)
414 + return AutoTokenizer.from_pretrained(pretrained_model_name_or_path,
415 + trust_remote_code=trust_remote_code)
416 +
417 +
418 ASYNC_REQUEST_FUNCS = {
419 "tgi": async_request_tgi,
420 "vllm": async_request_openai_completions,
benchmarks/benchmark_serving.py
+4 -1
@@ -39,7 +39,10 @@ from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
39 from tqdm.asyncio import tqdm
40 from transformers import PreTrainedTokenizerBase
41
42 -from vllm.transformers_utils.tokenizer import get_tokenizer
42 +try:
43 + from vllm.transformers_utils.tokenizer import get_tokenizer
44 +except ImportError:
45 + from backend_request_func import get_tokenizer
46
47
48 @dataclass