A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Use slow tokenizer for LLaMA (#84)
Woosuk Kwon committed
May 9, 2023 at 16:03 UTC
85eb6318391fc25d708af6109480498b45f56c77
3 files changed
+26
-5
cacheflow/frontend/fastapi_frontend.py
+2
-2
@@ -7,12 +7,12 @@ from typing import List, Dict, Optional
7
from fastapi import FastAPI, Request
8
from fastapi.responses import StreamingResponse
9
import ray
10
-from transformers import AutoTokenizer
10
import uvicorn
11
12
from cacheflow.core.server import (Server, add_server_arguments,
13
process_server_arguments,
14
initialize_cluster)
15
+from cacheflow.frontend.utils import get_tokenizer
16
from cacheflow.sampling_params import SamplingParams
17
from cacheflow.sequence import Sequence, SequenceGroup
18
from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
@@ -44,7 +44,7 @@ class FastAPIServer:
44
):
45
self.block_size = block_size
46
47
- self.tokenizer = AutoTokenizer.from_pretrained(model)
47
+ self.tokenizer = get_tokenizer(model)
48
self.seq_group_counter = Counter()
49
self.seq_counter = Counter()
50
if server_use_ray:
cacheflow/frontend/simple_frontend.py
+2
-3
@@ -1,8 +1,7 @@
1
import time
2
from typing import List, Optional, Tuple
3
4
-from transformers import AutoTokenizer
5
-
4
+from cacheflow.frontend.utils import get_tokenizer
5
from cacheflow.logger import init_logger
6
from cacheflow.sampling_params import SamplingParams
7
from cacheflow.sequence import Sequence, SequenceGroup
@@ -21,7 +20,7 @@ class SimpleFrontend:
20
) -> None:
21
self.block_size = block_size
22
24
- self.tokenizer = AutoTokenizer.from_pretrained(model_name)
23
+ self.tokenizer = get_tokenizer(model_name)
24
self.seq_group_counter = Counter()
25
self.seq_counter = Counter()
26
self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
cacheflow/frontend/utils.py
new
+22
@@ -0,0 +1,22 @@
1
+from typing import Union
2
+
3
+from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4
+ PreTrainedTokenizerFast)
5
+
6
+
7
+_MODEL_TYPES_WITH_SLOW_TOKENIZER = [
8
+ # LLaMA fast tokenizer has a bug related to protobuf.
9
+ # See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
10
+ "llama",
11
+]
12
+
13
+
14
+def get_tokenizer(
15
+ model_name: str,
16
+ *args,
17
+ **kwargs,
18
+) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
19
+ config = AutoConfig.from_pretrained(model_name)
20
+ if config.model_type in _MODEL_TYPES_WITH_SLOW_TOKENIZER:
21
+ kwargs["use_fast"] = False
22
+ return AutoTokenizer.from_pretrained(model_name, *args, **kwargs)