A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Frontend] Add tokenize/detokenize endpoints (#5054)
sasha0552 committed
Jun 26, 2024 at 16:54 UTC
c54269d967f2546868d9a52a10c110adc8f9822a
5 files changed
+143
-6
tests/entrypoints/test_openai_server.py
+49
@@ -9,6 +9,7 @@ import pytest
9
# using Ray for overall ease of process management, parallel requests,
10
# and debugging.
11
import ray
12
+import requests
13
import torch
14
# downloading lora to test lora requests
15
from huggingface_hub import snapshot_download
@@ -1366,5 +1367,53 @@ async def test_long_seed(client: openai.AsyncOpenAI):
1367
or "less_than_equal" in exc_info.value.message)
1368
1369
1370
+@pytest.mark.asyncio
1371
+@pytest.mark.parametrize(
1372
+ "model_name",
1373
+ [MODEL_NAME],
1374
+)
1375
+async def test_tokenize(server, client: openai.AsyncOpenAI, model_name: str):
1376
+ base_url = str(client.base_url)[:-3]
1377
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
1378
+
1379
+ for add_special in [False, True]:
1380
+ prompt = "This is a test prompt."
1381
+ tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
1382
+
1383
+ response = requests.post(base_url + "/tokenize",
1384
+ json={
1385
+ "add_special_tokens": add_special,
1386
+ "model": model_name,
1387
+ "prompt": prompt
1388
+ })
1389
+ response.raise_for_status()
1390
+ assert response.json() == {
1391
+ "tokens": tokens,
1392
+ "count": len(tokens),
1393
+ "max_model_len": 8192
1394
+ }
1395
+
1396
+
1397
+@pytest.mark.asyncio
1398
+@pytest.mark.parametrize(
1399
+ "model_name",
1400
+ [MODEL_NAME],
1401
+)
1402
+async def test_detokenize(server, client: openai.AsyncOpenAI, model_name: str):
1403
+ base_url = str(client.base_url)[:-3]
1404
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
1405
+
1406
+ prompt = "This is a test prompt."
1407
+ tokens = tokenizer.encode(prompt, add_special_tokens=False)
1408
+
1409
+ response = requests.post(base_url + "detokenize",
1410
+ json={
1411
+ "model": model_name,
1412
+ "tokens": tokens
1413
+ })
1414
+ response.raise_for_status()
1415
+ assert response.json() == {"prompt": prompt}
1416
+
1417
+
1418
if __name__ == "__main__":
1419
pytest.main([__file__])
vllm/entrypoints/openai/api_server.py
+30
-1
@@ -19,10 +19,17 @@ import vllm.envs as envs
19
from vllm.engine.arg_utils import AsyncEngineArgs
20
from vllm.engine.async_llm_engine import AsyncLLMEngine
21
from vllm.entrypoints.openai.cli_args import make_arg_parser
22
+# yapf conflicts with isort for this block
23
+# yapf: disable
24
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
25
ChatCompletionResponse,
26
CompletionRequest,
25
- EmbeddingRequest, ErrorResponse)
27
+ DetokenizeRequest,
28
+ DetokenizeResponse,
29
+ EmbeddingRequest, ErrorResponse,
30
+ TokenizeRequest,
31
+ TokenizeResponse)
32
+# yapf: enable
33
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
34
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
35
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
@@ -85,6 +92,28 @@ async def health() -> Response:
92
return Response(status_code=200)
93
94
95
+@app.post("/tokenize")
96
+async def tokenize(request: TokenizeRequest):
97
+ generator = await openai_serving_completion.create_tokenize(request)
98
+ if isinstance(generator, ErrorResponse):
99
+ return JSONResponse(content=generator.model_dump(),
100
+ status_code=generator.code)
101
+ else:
102
+ assert isinstance(generator, TokenizeResponse)
103
+ return JSONResponse(content=generator.model_dump())
104
+
105
+
106
+@app.post("/detokenize")
107
+async def detokenize(request: DetokenizeRequest):
108
+ generator = await openai_serving_completion.create_detokenize(request)
109
+ if isinstance(generator, ErrorResponse):
110
+ return JSONResponse(content=generator.model_dump(),
111
+ status_code=generator.code)
112
+ else:
113
+ assert isinstance(generator, DetokenizeResponse)
114
+ return JSONResponse(content=generator.model_dump())
115
+
116
+
117
@app.get("/v1/models")
118
async def show_available_models():
119
models = await openai_serving_chat.show_available_models()
vllm/entrypoints/openai/protocol.py
+21
@@ -699,3 +699,24 @@ class BatchRequestOutput(OpenAIBaseModel):
699
# For requests that failed with a non-HTTP error, this will contain more
700
# information on the cause of the failure.
701
error: Optional[Any]
702
+
703
+
704
+class TokenizeRequest(OpenAIBaseModel):
705
+ model: str
706
+ prompt: str
707
+ add_special_tokens: bool = Field(default=True)
708
+
709
+
710
+class TokenizeResponse(OpenAIBaseModel):
711
+ tokens: List[int]
712
+ count: int
713
+ max_model_len: int
714
+
715
+
716
+class DetokenizeRequest(OpenAIBaseModel):
717
+ model: str
718
+ tokens: List[int]
719
+
720
+
721
+class DetokenizeResponse(OpenAIBaseModel):
722
+ prompt: str
vllm/entrypoints/openai/serving_completion.py
+31
-1
@@ -16,7 +16,11 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
16
CompletionResponseChoice,
17
CompletionResponseStreamChoice,
18
CompletionStreamResponse,
19
- UsageInfo)
19
+ DetokenizeRequest,
20
+ DetokenizeResponse,
21
+ TokenizeRequest,
22
+ TokenizeResponse, UsageInfo)
23
+# yapf: enable
24
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
25
OpenAIServing)
26
from vllm.logger import init_logger
@@ -442,3 +446,29 @@ class OpenAIServingCompletion(OpenAIServing):
446
tokens=out_tokens,
447
top_logprobs=out_top_logprobs,
448
)
449
+
450
+ async def create_tokenize(self,
451
+ request: TokenizeRequest) -> TokenizeResponse:
452
+ error_check_ret = await self._check_model(request)
453
+ if error_check_ret is not None:
454
+ return error_check_ret
455
+
456
+ (input_ids, input_text) = self._validate_prompt_and_tokenize(
457
+ request,
458
+ prompt=request.prompt,
459
+ add_special_tokens=request.add_special_tokens)
460
+
461
+ return TokenizeResponse(tokens=input_ids,
462
+ count=len(input_ids),
463
+ max_model_len=self.max_model_len)
464
+
465
+ async def create_detokenize(
466
+ self, request: DetokenizeRequest) -> DetokenizeResponse:
467
+ error_check_ret = await self._check_model(request)
468
+ if error_check_ret is not None:
469
+ return error_check_ret
470
+
471
+ (input_ids, input_text) = self._validate_prompt_and_tokenize(
472
+ request, prompt_ids=request.tokens)
473
+
474
+ return DetokenizeResponse(prompt=input_text)
vllm/entrypoints/openai/serving_engine.py
+12
-4
@@ -10,9 +10,10 @@ from vllm.config import ModelConfig
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
CompletionRequest,
13
+ DetokenizeRequest,
14
EmbeddingRequest, ErrorResponse,
15
ModelCard, ModelList,
15
- ModelPermission)
16
+ ModelPermission, TokenizeRequest)
17
from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
19
from vllm.sequence import Logprob
@@ -99,8 +100,9 @@ class OpenAIServing:
100
return json_str
101
102
async def _check_model(
102
- self, request: Union[CompletionRequest, ChatCompletionRequest,
103
- EmbeddingRequest]
103
+ self, request: Union[ChatCompletionRequest, CompletionRequest,
104
+ DetokenizeRequest, EmbeddingRequest,
105
+ TokenizeRequest]
106
) -> Optional[ErrorResponse]:
107
if request.model in self.served_model_names:
108
return None
@@ -126,7 +128,8 @@ class OpenAIServing:
128
def _validate_prompt_and_tokenize(
129
self,
130
request: Union[ChatCompletionRequest, CompletionRequest,
129
- EmbeddingRequest],
131
+ DetokenizeRequest, EmbeddingRequest,
132
+ TokenizeRequest],
133
prompt: Optional[str] = None,
134
prompt_ids: Optional[List[int]] = None,
135
truncate_prompt_tokens: Optional[Annotated[int,
@@ -174,6 +177,11 @@ class OpenAIServing:
177
f"generation. Please reduce the length of the input.", )
178
return input_ids, input_text
179
180
+ # Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens
181
+ # and does not require model context length validation
182
+ if isinstance(request, (TokenizeRequest, DetokenizeRequest)):
183
+ return input_ids, input_text
184
+
185
if request.max_tokens is None:
186
if token_num >= self.max_model_len:
187
raise ValueError(