@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5689e256ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Frontend] Represent tokens with identifiable strings (#6626)

Evan Z. Liu committed Jul 24, 2024 at 18:51 UTC 5689e256baf0c45148a01ad147abf11ad82c9690
7 files changed +138 -19
tests/entrypoints/openai/test_completion.py
+7 -3
@@ -55,8 +55,9 @@ def zephyr_pa_files():
55
56
57 @pytest.fixture(scope="module")
58 -def server(zephyr_lora_files, zephyr_lora_added_tokens_files, zephyr_pa_files):
59 - args = [
58 +def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
59 + zephyr_pa_files):
60 + return [
61 # use half precision for speed and memory savings in CI environment
62 "--dtype",
63 "bfloat16",
@@ -85,7 +86,10 @@ def server(zephyr_lora_files, zephyr_lora_added_tokens_files, zephyr_pa_files):
86 "128",
87 ]
88
88 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
89 +
90 +@pytest.fixture(scope="module")
91 +def server(default_server_args):
92 + with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
93 yield remote_server
94
95
tests/entrypoints/openai/test_return_tokens_as_ids.py new
+83
@@ -0,0 +1,83 @@
1 +# Separate these tests out from test_completion and test_chat, because they
2 +# require launching a second server with a different flag. Running both servers
3 +# at the same time on a single node will OOM.
4 +
5 +import pytest
6 +
7 +from vllm.transformers_utils.tokenizer import get_tokenizer
8 +
9 +from ...utils import RemoteOpenAIServer
10 +from .test_completion import default_server_args # noqa: F401
11 +from .test_completion import zephyr_lora_added_tokens_files # noqa: F401
12 +from .test_completion import zephyr_lora_files # noqa: F401
13 +from .test_completion import zephyr_pa_files # noqa: F401
14 +from .test_completion import MODEL_NAME
15 +
16 +
17 +@pytest.fixture(scope="module")
18 +def server_with_return_tokens_as_token_ids_flag(
19 + default_server_args): # noqa: F811
20 + args_with_flag = default_server_args + ["--return-tokens-as-token-ids"]
21 + with RemoteOpenAIServer(MODEL_NAME, args_with_flag) as remote_server:
22 + yield remote_server
23 +
24 +
25 +@pytest.mark.asyncio
26 +async def test_completion_return_tokens_as_token_ids_completion(
27 + server_with_return_tokens_as_token_ids_flag):
28 + client = server_with_return_tokens_as_token_ids_flag.get_async_client()
29 +
30 + completion = await client.completions.create(
31 + model=MODEL_NAME,
32 + # Include Unicode characters to test for dividing a single
33 + # character across multiple tokens: 🎉 is [28705, 31862] for the
34 + # Zephyr tokenizer
35 + prompt="Say 'Hello, world! 🎉'",
36 + echo=True,
37 + temperature=0,
38 + max_tokens=10,
39 + logprobs=1)
40 +
41 + text = completion.choices[0].text
42 + token_strs = completion.choices[0].logprobs.tokens
43 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
44 + # Check that the token representations are consistent between raw tokens
45 + # and top_logprobs
46 + # Slice off the first one, because there's no scoring associated with BOS
47 + top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
48 + top_logprob_keys = [
49 + next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
50 + ]
51 + assert token_strs[1:] == top_logprob_keys
52 +
53 + # Check that decoding the tokens gives the expected text
54 + tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
55 + assert text == tokenizer.decode(tokens, skip_special_tokens=True)
56 +
57 +
58 +@pytest.mark.asyncio
59 +async def test_chat_return_tokens_as_token_ids_completion(
60 + server_with_return_tokens_as_token_ids_flag):
61 + client = server_with_return_tokens_as_token_ids_flag.get_async_client()
62 + response = await client.chat.completions.create(
63 + model=MODEL_NAME,
64 + # Include Unicode characters to test for dividing a single
65 + # character across multiple tokens: 🎉 is [28705, 31862] for the
66 + # Zephyr tokenizer
67 + messages=[{
68 + "role": "system",
69 + "content": "You like to respond in only emojis, like 🎉"
70 + }, {
71 + "role": "user",
72 + "content": "Please write some emojis: 🐱🐶🎉"
73 + }],
74 + temperature=0,
75 + max_tokens=8,
76 + logprobs=True)
77 +
78 + text = response.choices[0].message.content
79 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
80 + token_ids = []
81 + for logprob_content in response.choices[0].logprobs.content:
82 + token_ids.append(int(logprob_content.token.removeprefix("token_id:")))
83 + assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
vllm/entrypoints/openai/api_server.py
+2
@@ -254,6 +254,7 @@ async def build_server(
254 prompt_adapters=args.prompt_adapters,
255 request_logger=request_logger,
256 chat_template=args.chat_template,
257 + return_tokens_as_token_ids=args.return_tokens_as_token_ids,
258 )
259 openai_serving_completion = OpenAIServingCompletion(
260 engine,
@@ -262,6 +263,7 @@ async def build_server(
263 lora_modules=args.lora_modules,
264 prompt_adapters=args.prompt_adapters,
265 request_logger=request_logger,
266 + return_tokens_as_token_ids=args.return_tokens_as_token_ids,
267 )
268 openai_serving_embedding = OpenAIServingEmbedding(
269 engine,
vllm/entrypoints/openai/cli_args.py
+6
@@ -128,6 +128,12 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
128 "using @app.middleware('http'). "
129 "If a class is provided, vLLM will add it to the server "
130 "using app.add_middleware(). ")
131 + parser.add_argument(
132 + "--return-tokens-as-token-ids",
133 + action="store_true",
134 + help="When --max-logprobs is specified, represents single tokens as"
135 + "strings of the form 'token_id:{token_id}' so that tokens that"
136 + "are not JSON-encodable can be identified.")
137
138 parser = AsyncEngineArgs.add_cli_args(parser)
139
vllm/entrypoints/openai/serving_chat.py
+16 -7
@@ -50,13 +50,15 @@ class OpenAIServingChat(OpenAIServing):
50 prompt_adapters: Optional[List[PromptAdapterPath]],
51 request_logger: Optional[RequestLogger],
52 chat_template: Optional[str],
53 + return_tokens_as_token_ids: bool = False,
54 ):
55 super().__init__(engine=engine,
56 model_config=model_config,
57 served_model_names=served_model_names,
58 lora_modules=lora_modules,
59 prompt_adapters=prompt_adapters,
59 - request_logger=request_logger)
60 + request_logger=request_logger,
61 + return_tokens_as_token_ids=return_tokens_as_token_ids)
62
63 self.response_role = response_role
64
@@ -522,11 +524,14 @@ class OpenAIServingChat(OpenAIServing):
524 self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
525 tokenizer: PreTrainedTokenizer) -> List[ChatCompletionLogProb]:
526 return [
525 - ChatCompletionLogProb(
526 - token=(token := self._get_decoded_token(p[1], p[0],
527 - tokenizer)),
528 - logprob=max(p[1].logprob, -9999.0),
529 - bytes=list(token.encode("utf-8", errors="replace")))
527 + ChatCompletionLogProb(token=(token := self._get_decoded_token(
528 + p[1],
529 + p[0],
530 + tokenizer,
531 + return_as_token_id=self.return_tokens_as_token_ids)),
532 + logprob=max(p[1].logprob, -9999.0),
533 + bytes=list(
534 + token.encode("utf-8", errors="replace")))
535 for i, p in enumerate(logprobs.items())
536 if top_logprobs and i < top_logprobs
537 ]
@@ -546,6 +551,8 @@ class OpenAIServingChat(OpenAIServing):
551 step_top_logprobs = top_logprobs[i]
552 if step_top_logprobs is None:
553 token = tokenizer.decode(token_id)
554 + if self.return_tokens_as_token_ids:
555 + token = f"token_id:{token_id}"
556 logprobs_content.append(
557 ChatCompletionLogProbsContent(
558 token=token,
@@ -553,7 +560,9 @@ class OpenAIServingChat(OpenAIServing):
560 else:
561 logprobs_content.append(
562 ChatCompletionLogProbsContent(
556 - token=step_top_logprobs[token_id].decoded_token,
563 + token=self._get_decoded_token(
564 + step_top_logprobs[token_id], token_id, tokenizer,
565 + self.return_tokens_as_token_ids),
566 logprob=max(step_top_logprobs[token_id].logprob,
567 -9999.0),
568 bytes=list(
vllm/entrypoints/openai/serving_completion.py
+15 -4
@@ -51,13 +51,15 @@ class OpenAIServingCompletion(OpenAIServing):
51 lora_modules: Optional[List[LoRAModulePath]],
52 prompt_adapters: Optional[List[PromptAdapterPath]],
53 request_logger: Optional[RequestLogger],
54 + return_tokens_as_token_ids: bool = False,
55 ):
56 super().__init__(engine=engine,
57 model_config=model_config,
58 served_model_names=served_model_names,
59 lora_modules=lora_modules,
60 prompt_adapters=prompt_adapters,
60 - request_logger=request_logger)
61 + request_logger=request_logger,
62 + return_tokens_as_token_ids=return_tokens_as_token_ids)
63
64 async def create_completion(self, request: CompletionRequest,
65 raw_request: Request):
@@ -430,12 +432,17 @@ class OpenAIServingCompletion(OpenAIServing):
432 step_top_logprobs = top_logprobs[i]
433 if step_top_logprobs is None:
434 token = tokenizer.decode(token_id)
435 + if self.return_tokens_as_token_ids:
436 + token = f"token_id:{token_id}"
437 out_tokens.append(token)
438 out_token_logprobs.append(None)
439 out_top_logprobs.append(None)
440 else:
437 - token = self._get_decoded_token(step_top_logprobs[token_id],
438 - token_id, tokenizer)
441 + token = self._get_decoded_token(
442 + step_top_logprobs[token_id],
443 + token_id,
444 + tokenizer,
445 + return_as_token_id=self.return_tokens_as_token_ids)
446 token_logprob = max(step_top_logprobs[token_id].logprob,
447 -9999.0)
448 out_tokens.append(token)
@@ -448,7 +455,11 @@ class OpenAIServingCompletion(OpenAIServing):
455 out_top_logprobs.append({
456 # Convert float("-inf") to the
457 # JSON-serializable float that OpenAI uses
451 - self._get_decoded_token(top_lp[1], top_lp[0], tokenizer):
458 + self._get_decoded_token(
459 + top_lp[1],
460 + top_lp[0],
461 + tokenizer,
462 + return_as_token_id=self.return_tokens_as_token_ids):
463 max(top_lp[1].logprob, -9999.0)
464 for i, top_lp in enumerate(step_top_logprobs.items())
465 if num_output_top_logprobs >= i
vllm/entrypoints/openai/serving_engine.py
+9 -5
@@ -68,6 +68,7 @@ class OpenAIServing:
68 lora_modules: Optional[List[LoRAModulePath]],
69 prompt_adapters: Optional[List[PromptAdapterPath]],
70 request_logger: Optional[RequestLogger],
71 + return_tokens_as_token_ids: bool = False,
72 ):
73 super().__init__()
74
@@ -102,6 +103,7 @@ class OpenAIServing:
103 prompt_adapter_num_virtual_tokens=num_virtual_tokens))
104
105 self.request_logger = request_logger
106 + self.return_tokens_as_token_ids = return_tokens_as_token_ids
107
108 async def show_available_models(self) -> ModelList:
109 """Show available models. Right now we only have one model."""
@@ -384,11 +386,13 @@ class OpenAIServing:
386 )
387
388 @staticmethod
387 - def _get_decoded_token(
388 - logprob: Logprob,
389 - token_id: int,
390 - tokenizer: AnyTokenizer,
391 - ) -> str:
389 + def _get_decoded_token(logprob: Logprob,
390 + token_id: int,
391 + tokenizer: AnyTokenizer,
392 + return_as_token_id: bool = False) -> str:
393 + if return_as_token_id:
394 + return f"token_id:{token_id}"
395 +
396 if logprob.decoded_token is not None:
397 return logprob.decoded_token
398 return tokenizer.decode(token_id)