A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix][Core] Fix get decoding config from ray (#4335)
Roy committed
Apr 27, 2024 at 19:30 UTC
7134303cbbb7c82cdfcb0c87d59bb48fe6ad642b
6 files changed
+174
-3
tests/async_engine/test_async_llm_engine.py
+2
@@ -91,4 +91,6 @@ async def test_new_requests_event():
91
assert engine.engine.step_calls == old_step_calls + 1
92
93
engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
94
+ assert engine.get_model_config() is not None
95
assert engine.get_tokenizer() is not None
96
+ assert engine.get_decoding_config() is not None
tests/async_engine/test_openapi_server_ray.py
new
+157
@@ -0,0 +1,157 @@
1
+# imports for guided decoding tests
2
+import os
3
+import subprocess
4
+import sys
5
+import time
6
+
7
+import openai # use the official client for correctness check
8
+import pytest
9
+# using Ray for overall ease of process management, parallel requests,
10
+# and debugging.
11
+import ray
12
+import requests
13
+
14
+MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
15
+# any model with a chat template should work here
16
+MODEL_NAME = "facebook/opt-125m"
17
+
18
+
19
+@ray.remote(num_gpus=1)
20
+class ServerRunner:
21
+
22
+ def __init__(self, args):
23
+ env = os.environ.copy()
24
+ env["PYTHONUNBUFFERED"] = "1"
25
+ self.proc = subprocess.Popen(
26
+ ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
27
+ env=env,
28
+ stdout=sys.stdout,
29
+ stderr=sys.stderr,
30
+ )
31
+ self._wait_for_server()
32
+
33
+ def ready(self):
34
+ return True
35
+
36
+ def _wait_for_server(self):
37
+ # run health check
38
+ start = time.time()
39
+ while True:
40
+ try:
41
+ if requests.get(
42
+ "http://localhost:8000/health").status_code == 200:
43
+ break
44
+ except Exception as err:
45
+ if self.proc.poll() is not None:
46
+ raise RuntimeError("Server exited unexpectedly.") from err
47
+
48
+ time.sleep(0.5)
49
+ if time.time() - start > MAX_SERVER_START_WAIT_S:
50
+ raise RuntimeError(
51
+ "Server failed to start in time.") from err
52
+
53
+ def __del__(self):
54
+ if hasattr(self, "proc"):
55
+ self.proc.terminate()
56
+
57
+
58
+@pytest.fixture(scope="session")
59
+def server():
60
+ ray.init()
61
+ server_runner = ServerRunner.remote([
62
+ "--model",
63
+ MODEL_NAME,
64
+ # use half precision for speed and memory savings in CI environment
65
+ "--dtype",
66
+ "float16",
67
+ "--max-model-len",
68
+ "2048",
69
+ "--enforce-eager",
70
+ "--engine-use-ray"
71
+ ])
72
+ ray.get(server_runner.ready.remote())
73
+ yield server_runner
74
+ ray.shutdown()
75
+
76
+
77
+@pytest.fixture(scope="session")
78
+def client():
79
+ client = openai.AsyncOpenAI(
80
+ base_url="http://localhost:8000/v1",
81
+ api_key="token-abc123",
82
+ )
83
+ yield client
84
+
85
+
86
+@pytest.mark.asyncio
87
+async def test_check_models(server, client: openai.AsyncOpenAI):
88
+ models = await client.models.list()
89
+ models = models.data
90
+ served_model = models[0]
91
+ assert served_model.id == MODEL_NAME
92
+ assert all(model.root == MODEL_NAME for model in models)
93
+
94
+
95
+@pytest.mark.asyncio
96
+async def test_single_completion(server, client: openai.AsyncOpenAI):
97
+ completion = await client.completions.create(model=MODEL_NAME,
98
+ prompt="Hello, my name is",
99
+ max_tokens=5,
100
+ temperature=0.0)
101
+
102
+ assert completion.id is not None
103
+ assert completion.choices is not None and len(completion.choices) == 1
104
+ assert completion.choices[0].text is not None and len(
105
+ completion.choices[0].text) >= 5
106
+ assert completion.choices[0].finish_reason == "length"
107
+ assert completion.usage == openai.types.CompletionUsage(
108
+ completion_tokens=5, prompt_tokens=6, total_tokens=11)
109
+
110
+ # test using token IDs
111
+ completion = await client.completions.create(
112
+ model=MODEL_NAME,
113
+ prompt=[0, 0, 0, 0, 0],
114
+ max_tokens=5,
115
+ temperature=0.0,
116
+ )
117
+ assert completion.choices[0].text is not None and len(
118
+ completion.choices[0].text) >= 5
119
+
120
+
121
+@pytest.mark.asyncio
122
+async def test_single_chat_session(server, client: openai.AsyncOpenAI):
123
+ messages = [{
124
+ "role": "system",
125
+ "content": "you are a helpful assistant"
126
+ }, {
127
+ "role": "user",
128
+ "content": "what is 1+1?"
129
+ }]
130
+
131
+ # test single completion
132
+ chat_completion = await client.chat.completions.create(model=MODEL_NAME,
133
+ messages=messages,
134
+ max_tokens=10,
135
+ logprobs=True,
136
+ top_logprobs=5)
137
+ assert chat_completion.id is not None
138
+ assert chat_completion.choices is not None and len(
139
+ chat_completion.choices) == 1
140
+ assert chat_completion.choices[0].message is not None
141
+ assert chat_completion.choices[0].logprobs is not None
142
+ assert chat_completion.choices[0].logprobs.top_logprobs is not None
143
+ assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
144
+ message = chat_completion.choices[0].message
145
+ assert message.content is not None and len(message.content) >= 10
146
+ assert message.role == "assistant"
147
+ messages.append({"role": "assistant", "content": message.content})
148
+
149
+ # test multi-turn dialogue
150
+ messages.append({"role": "user", "content": "express your result in json"})
151
+ chat_completion = await client.chat.completions.create(
152
+ model=MODEL_NAME,
153
+ messages=messages,
154
+ max_tokens=10,
155
+ )
156
+ message = chat_completion.choices[0].message
157
+ assert message.content is not None and len(message.content) >= 0
vllm/engine/async_llm_engine.py
+9
-1
@@ -7,7 +7,7 @@ from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
7
8
from transformers import PreTrainedTokenizer
9
10
-from vllm.config import ModelConfig
10
+from vllm.config import DecodingConfig, ModelConfig
11
from vllm.engine.arg_utils import AsyncEngineArgs
12
from vllm.engine.llm_engine import LLMEngine
13
from vllm.executor.ray_utils import initialize_ray_cluster, ray
@@ -697,6 +697,14 @@ class AsyncLLMEngine:
697
else:
698
return self.engine.get_model_config()
699
700
+ async def get_decoding_config(self) -> DecodingConfig:
701
+ """Get the decoding configuration of the vLLM engine."""
702
+ if self.engine_use_ray:
703
+ return await self.engine.get_decoding_config.remote( # type: ignore
704
+ )
705
+ else:
706
+ return self.engine.get_decoding_config()
707
+
708
async def do_log_stats(self) -> None:
709
if self.engine_use_ray:
710
await self.engine.do_log_stats.remote() # type: ignore
vllm/engine/llm_engine.py
+4
@@ -467,6 +467,10 @@ class LLMEngine:
467
"""Gets the model configuration."""
468
return self.model_config
469
470
+ def get_decoding_config(self) -> DecodingConfig:
471
+ """Gets the decoding configuration."""
472
+ return self.decoding_config
473
+
474
def get_num_unfinished_requests(self) -> int:
475
"""Gets the number of unfinished requests."""
476
return self.scheduler.get_num_unfinished_seq_groups()
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -101,7 +101,7 @@ class OpenAIServingChat(OpenAIServing):
101
request, prompt=prompt)
102
sampling_params = request.to_sampling_params()
103
lora_request = self._maybe_get_lora(request)
104
- decoding_config = self.engine.engine.decoding_config
104
+ decoding_config = await self.engine.get_decoding_config()
105
guided_decoding_backend = request.guided_decoding_backend \
106
or decoding_config.guided_decoding_backend
107
guided_decode_logits_processor = (
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -89,7 +89,7 @@ class OpenAIServingCompletion(OpenAIServing):
89
try:
90
sampling_params = request.to_sampling_params()
91
lora_request = self._maybe_get_lora(request)
92
- decoding_config = self.engine.engine.decoding_config
92
+ decoding_config = await self.engine.get_decoding_config()
93
guided_decoding_backend = request.guided_decoding_backend \
94
or decoding_config.guided_decoding_backend
95
guided_decode_logit_processor = (