A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Frontend] Represent tokens with identifiable strings (#6626)
Evan Z. Liu committed
Jul 24, 2024 at 18:51 UTC
5689e256baf0c45148a01ad147abf11ad82c9690
7 files changed
+138
-19
tests/entrypoints/openai/test_completion.py
+7
-3
@@ -55,8 +55,9 @@ def zephyr_pa_files():
55
56
57
@pytest.fixture(scope="module")
58
-def server(zephyr_lora_files, zephyr_lora_added_tokens_files, zephyr_pa_files):
59
- args = [
58
+def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
59
+ zephyr_pa_files):
60
+ return [
61
# use half precision for speed and memory savings in CI environment
62
"--dtype",
63
"bfloat16",
@@ -85,7 +86,10 @@ def server(zephyr_lora_files, zephyr_lora_added_tokens_files, zephyr_pa_files):
86
"128",
87
]
88
88
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
89
+
90
+@pytest.fixture(scope="module")
91
+def server(default_server_args):
92
+ with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
93
yield remote_server
94
95
tests/entrypoints/openai/test_return_tokens_as_ids.py
new
+83
@@ -0,0 +1,83 @@
1
+# Separate these tests out from test_completion and test_chat, because they
2
+# require launching a second server with a different flag. Running both servers
3
+# at the same time on a single node will OOM.
4
+
5
+import pytest
6
+
7
+from vllm.transformers_utils.tokenizer import get_tokenizer
8
+
9
+from ...utils import RemoteOpenAIServer
10
+from .test_completion import default_server_args # noqa: F401
11
+from .test_completion import zephyr_lora_added_tokens_files # noqa: F401
12
+from .test_completion import zephyr_lora_files # noqa: F401
13
+from .test_completion import zephyr_pa_files # noqa: F401
14
+from .test_completion import MODEL_NAME
15
+
16
+
17
+@pytest.fixture(scope="module")
18
+def server_with_return_tokens_as_token_ids_flag(
19
+ default_server_args): # noqa: F811
20
+ args_with_flag = default_server_args + ["--return-tokens-as-token-ids"]
21
+ with RemoteOpenAIServer(MODEL_NAME, args_with_flag) as remote_server:
22
+ yield remote_server
23
+
24
+
25
+@pytest.mark.asyncio
26
+async def test_completion_return_tokens_as_token_ids_completion(
27
+ server_with_return_tokens_as_token_ids_flag):
28
+ client = server_with_return_tokens_as_token_ids_flag.get_async_client()
29
+
30
+ completion = await client.completions.create(
31
+ model=MODEL_NAME,
32
+ # Include Unicode characters to test for dividing a single
33
+ # character across multiple tokens: 🎉 is [28705, 31862] for the
34
+ # Zephyr tokenizer
35
+ prompt="Say 'Hello, world! 🎉'",
36
+ echo=True,
37
+ temperature=0,
38
+ max_tokens=10,
39
+ logprobs=1)
40
+
41
+ text = completion.choices[0].text
42
+ token_strs = completion.choices[0].logprobs.tokens
43
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
44
+ # Check that the token representations are consistent between raw tokens
45
+ # and top_logprobs
46
+ # Slice off the first one, because there's no scoring associated with BOS
47
+ top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
48
+ top_logprob_keys = [
49
+ next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
50
+ ]
51
+ assert token_strs[1:] == top_logprob_keys
52
+
53
+ # Check that decoding the tokens gives the expected text
54
+ tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
55
+ assert text == tokenizer.decode(tokens, skip_special_tokens=True)
56
+
57
+
58
+@pytest.mark.asyncio
59
+async def test_chat_return_tokens_as_token_ids_completion(
60
+ server_with_return_tokens_as_token_ids_flag):
61
+ client = server_with_return_tokens_as_token_ids_flag.get_async_client()
62
+ response = await client.chat.completions.create(
63
+ model=MODEL_NAME,
64
+ # Include Unicode characters to test for dividing a single
65
+ # character across multiple tokens: 🎉 is [28705, 31862] for the
66
+ # Zephyr tokenizer
67
+ messages=[{
68
+ "role": "system",
69
+ "content": "You like to respond in only emojis, like 🎉"
70
+ }, {
71
+ "role": "user",
72
+ "content": "Please write some emojis: 🐱🐶🎉"
73
+ }],
74
+ temperature=0,
75
+ max_tokens=8,
76
+ logprobs=True)
77
+
78
+ text = response.choices[0].message.content
79
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
80
+ token_ids = []
81
+ for logprob_content in response.choices[0].logprobs.content:
82
+ token_ids.append(int(logprob_content.token.removeprefix("token_id:")))
83
+ assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
vllm/entrypoints/openai/api_server.py
+2
@@ -254,6 +254,7 @@ async def build_server(
254
prompt_adapters=args.prompt_adapters,
255
request_logger=request_logger,
256
chat_template=args.chat_template,
257
+ return_tokens_as_token_ids=args.return_tokens_as_token_ids,
258
)
259
openai_serving_completion = OpenAIServingCompletion(
260
engine,
@@ -262,6 +263,7 @@ async def build_server(
263
lora_modules=args.lora_modules,
264
prompt_adapters=args.prompt_adapters,
265
request_logger=request_logger,
266
+ return_tokens_as_token_ids=args.return_tokens_as_token_ids,
267
)
268
openai_serving_embedding = OpenAIServingEmbedding(
269
engine,
vllm/entrypoints/openai/cli_args.py
+6
@@ -128,6 +128,12 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
128
"using @app.middleware('http'). "
129
"If a class is provided, vLLM will add it to the server "
130
"using app.add_middleware(). ")
131
+ parser.add_argument(
132
+ "--return-tokens-as-token-ids",
133
+ action="store_true",
134
+ help="When --max-logprobs is specified, represents single tokens as"
135
+ "strings of the form 'token_id:{token_id}' so that tokens that"
136
+ "are not JSON-encodable can be identified.")
137
138
parser = AsyncEngineArgs.add_cli_args(parser)
139
vllm/entrypoints/openai/serving_chat.py
+16
-7
@@ -50,13 +50,15 @@ class OpenAIServingChat(OpenAIServing):
50
prompt_adapters: Optional[List[PromptAdapterPath]],
51
request_logger: Optional[RequestLogger],
52
chat_template: Optional[str],
53
+ return_tokens_as_token_ids: bool = False,
54
):
55
super().__init__(engine=engine,
56
model_config=model_config,
57
served_model_names=served_model_names,
58
lora_modules=lora_modules,
59
prompt_adapters=prompt_adapters,
59
- request_logger=request_logger)
60
+ request_logger=request_logger,
61
+ return_tokens_as_token_ids=return_tokens_as_token_ids)
62
63
self.response_role = response_role
64
@@ -522,11 +524,14 @@ class OpenAIServingChat(OpenAIServing):
524
self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
525
tokenizer: PreTrainedTokenizer) -> List[ChatCompletionLogProb]:
526
return [
525
- ChatCompletionLogProb(
526
- token=(token := self._get_decoded_token(p[1], p[0],
527
- tokenizer)),
528
- logprob=max(p[1].logprob, -9999.0),
529
- bytes=list(token.encode("utf-8", errors="replace")))
527
+ ChatCompletionLogProb(token=(token := self._get_decoded_token(
528
+ p[1],
529
+ p[0],
530
+ tokenizer,
531
+ return_as_token_id=self.return_tokens_as_token_ids)),
532
+ logprob=max(p[1].logprob, -9999.0),
533
+ bytes=list(
534
+ token.encode("utf-8", errors="replace")))
535
for i, p in enumerate(logprobs.items())
536
if top_logprobs and i < top_logprobs
537
]
@@ -546,6 +551,8 @@ class OpenAIServingChat(OpenAIServing):
551
step_top_logprobs = top_logprobs[i]
552
if step_top_logprobs is None:
553
token = tokenizer.decode(token_id)
554
+ if self.return_tokens_as_token_ids:
555
+ token = f"token_id:{token_id}"
556
logprobs_content.append(
557
ChatCompletionLogProbsContent(
558
token=token,
@@ -553,7 +560,9 @@ class OpenAIServingChat(OpenAIServing):
560
else:
561
logprobs_content.append(
562
ChatCompletionLogProbsContent(
556
- token=step_top_logprobs[token_id].decoded_token,
563
+ token=self._get_decoded_token(
564
+ step_top_logprobs[token_id], token_id, tokenizer,
565
+ self.return_tokens_as_token_ids),
566
logprob=max(step_top_logprobs[token_id].logprob,
567
-9999.0),
568
bytes=list(
vllm/entrypoints/openai/serving_completion.py
+15
-4
@@ -51,13 +51,15 @@ class OpenAIServingCompletion(OpenAIServing):
51
lora_modules: Optional[List[LoRAModulePath]],
52
prompt_adapters: Optional[List[PromptAdapterPath]],
53
request_logger: Optional[RequestLogger],
54
+ return_tokens_as_token_ids: bool = False,
55
):
56
super().__init__(engine=engine,
57
model_config=model_config,
58
served_model_names=served_model_names,
59
lora_modules=lora_modules,
60
prompt_adapters=prompt_adapters,
60
- request_logger=request_logger)
61
+ request_logger=request_logger,
62
+ return_tokens_as_token_ids=return_tokens_as_token_ids)
63
64
async def create_completion(self, request: CompletionRequest,
65
raw_request: Request):
@@ -430,12 +432,17 @@ class OpenAIServingCompletion(OpenAIServing):
432
step_top_logprobs = top_logprobs[i]
433
if step_top_logprobs is None:
434
token = tokenizer.decode(token_id)
435
+ if self.return_tokens_as_token_ids:
436
+ token = f"token_id:{token_id}"
437
out_tokens.append(token)
438
out_token_logprobs.append(None)
439
out_top_logprobs.append(None)
440
else:
437
- token = self._get_decoded_token(step_top_logprobs[token_id],
438
- token_id, tokenizer)
441
+ token = self._get_decoded_token(
442
+ step_top_logprobs[token_id],
443
+ token_id,
444
+ tokenizer,
445
+ return_as_token_id=self.return_tokens_as_token_ids)
446
token_logprob = max(step_top_logprobs[token_id].logprob,
447
-9999.0)
448
out_tokens.append(token)
@@ -448,7 +455,11 @@ class OpenAIServingCompletion(OpenAIServing):
455
out_top_logprobs.append({
456
# Convert float("-inf") to the
457
# JSON-serializable float that OpenAI uses
451
- self._get_decoded_token(top_lp[1], top_lp[0], tokenizer):
458
+ self._get_decoded_token(
459
+ top_lp[1],
460
+ top_lp[0],
461
+ tokenizer,
462
+ return_as_token_id=self.return_tokens_as_token_ids):
463
max(top_lp[1].logprob, -9999.0)
464
for i, top_lp in enumerate(step_top_logprobs.items())
465
if num_output_top_logprobs >= i
vllm/entrypoints/openai/serving_engine.py
+9
-5
@@ -68,6 +68,7 @@ class OpenAIServing:
68
lora_modules: Optional[List[LoRAModulePath]],
69
prompt_adapters: Optional[List[PromptAdapterPath]],
70
request_logger: Optional[RequestLogger],
71
+ return_tokens_as_token_ids: bool = False,
72
):
73
super().__init__()
74
@@ -102,6 +103,7 @@ class OpenAIServing:
103
prompt_adapter_num_virtual_tokens=num_virtual_tokens))
104
105
self.request_logger = request_logger
106
+ self.return_tokens_as_token_ids = return_tokens_as_token_ids
107
108
async def show_available_models(self) -> ModelList:
109
"""Show available models. Right now we only have one model."""
@@ -384,11 +386,13 @@ class OpenAIServing:
386
)
387
388
@staticmethod
387
- def _get_decoded_token(
388
- logprob: Logprob,
389
- token_id: int,
390
- tokenizer: AnyTokenizer,
391
- ) -> str:
389
+ def _get_decoded_token(logprob: Logprob,
390
+ token_id: int,
391
+ tokenizer: AnyTokenizer,
392
+ return_as_token_id: bool = False) -> str:
393
+ if return_as_token_id:
394
+ return f"token_id:{token_id}"
395
+
396
if logprob.decoded_token is not None:
397
return logprob.decoded_token
398
return tokenizer.decode(token_id)