@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 85eb631839
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Use slow tokenizer for LLaMA (#84)

Woosuk Kwon committed May 9, 2023 at 16:03 UTC 85eb6318391fc25d708af6109480498b45f56c77
3 files changed +26 -5
cacheflow/frontend/fastapi_frontend.py
+2 -2
@@ -7,12 +7,12 @@ from typing import List, Dict, Optional
7 from fastapi import FastAPI, Request
8 from fastapi.responses import StreamingResponse
9 import ray
10 -from transformers import AutoTokenizer
10 import uvicorn
11
12 from cacheflow.core.server import (Server, add_server_arguments,
13 process_server_arguments,
14 initialize_cluster)
15 +from cacheflow.frontend.utils import get_tokenizer
16 from cacheflow.sampling_params import SamplingParams
17 from cacheflow.sequence import Sequence, SequenceGroup
18 from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
@@ -44,7 +44,7 @@ class FastAPIServer:
44 ):
45 self.block_size = block_size
46
47 - self.tokenizer = AutoTokenizer.from_pretrained(model)
47 + self.tokenizer = get_tokenizer(model)
48 self.seq_group_counter = Counter()
49 self.seq_counter = Counter()
50 if server_use_ray:
cacheflow/frontend/simple_frontend.py
+2 -3
@@ -1,8 +1,7 @@
1 import time
2 from typing import List, Optional, Tuple
3
4 -from transformers import AutoTokenizer
5 -
4 +from cacheflow.frontend.utils import get_tokenizer
5 from cacheflow.logger import init_logger
6 from cacheflow.sampling_params import SamplingParams
7 from cacheflow.sequence import Sequence, SequenceGroup
@@ -21,7 +20,7 @@ class SimpleFrontend:
20 ) -> None:
21 self.block_size = block_size
22
24 - self.tokenizer = AutoTokenizer.from_pretrained(model_name)
23 + self.tokenizer = get_tokenizer(model_name)
24 self.seq_group_counter = Counter()
25 self.seq_counter = Counter()
26 self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
cacheflow/frontend/utils.py new
+22
@@ -0,0 +1,22 @@
1 +from typing import Union
2 +
3 +from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4 + PreTrainedTokenizerFast)
5 +
6 +
7 +_MODEL_TYPES_WITH_SLOW_TOKENIZER = [
8 + # LLaMA fast tokenizer has a bug related to protobuf.
9 + # See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
10 + "llama",
11 +]
12 +
13 +
14 +def get_tokenizer(
15 + model_name: str,
16 + *args,
17 + **kwargs,
18 +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
19 + config = AutoConfig.from_pretrained(model_name)
20 + if config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
21 + kwargs["use_fast"] = False
22 + return AutoTokenizer.from_pretrained(model_name, *args, **kwargs)