@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7134303cbb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix][Core] Fix get decoding config from ray (#4335)

Roy committed Apr 27, 2024 at 19:30 UTC 7134303cbbb7c82cdfcb0c87d59bb48fe6ad642b
6 files changed +174 -3
tests/async_engine/test_async_llm_engine.py
+2
@@ -91,4 +91,6 @@ async def test_new_requests_event():
91 assert engine.engine.step_calls == old_step_calls + 1
92
93 engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
94 + assert engine.get_model_config() is not None
95 assert engine.get_tokenizer() is not None
96 + assert engine.get_decoding_config() is not None
tests/async_engine/test_openapi_server_ray.py new
+157
@@ -0,0 +1,157 @@
1 +# imports for guided decoding tests
2 +import os
3 +import subprocess
4 +import sys
5 +import time
6 +
7 +import openai # use the official client for correctness check
8 +import pytest
9 +# using Ray for overall ease of process management, parallel requests,
10 +# and debugging.
11 +import ray
12 +import requests
13 +
14 +MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
15 +# any model with a chat template should work here
16 +MODEL_NAME = "facebook/opt-125m"
17 +
18 +
19 +@ray.remote(num_gpus=1)
20 +class ServerRunner:
21 +
22 + def __init__(self, args):
23 + env = os.environ.copy()
24 + env["PYTHONUNBUFFERED"] = "1"
25 + self.proc = subprocess.Popen(
26 + ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
27 + env=env,
28 + stdout=sys.stdout,
29 + stderr=sys.stderr,
30 + )
31 + self._wait_for_server()
32 +
33 + def ready(self):
34 + return True
35 +
36 + def _wait_for_server(self):
37 + # run health check
38 + start = time.time()
39 + while True:
40 + try:
41 + if requests.get(
42 + "http://localhost:8000/health").status_code == 200:
43 + break
44 + except Exception as err:
45 + if self.proc.poll() is not None:
46 + raise RuntimeError("Server exited unexpectedly.") from err
47 +
48 + time.sleep(0.5)
49 + if time.time() - start > MAX_SERVER_START_WAIT_S:
50 + raise RuntimeError(
51 + "Server failed to start in time.") from err
52 +
53 + def __del__(self):
54 + if hasattr(self, "proc"):
55 + self.proc.terminate()
56 +
57 +
58 +@pytest.fixture(scope="session")
59 +def server():
60 + ray.init()
61 + server_runner = ServerRunner.remote([
62 + "--model",
63 + MODEL_NAME,
64 + # use half precision for speed and memory savings in CI environment
65 + "--dtype",
66 + "float16",
67 + "--max-model-len",
68 + "2048",
69 + "--enforce-eager",
70 + "--engine-use-ray"
71 + ])
72 + ray.get(server_runner.ready.remote())
73 + yield server_runner
74 + ray.shutdown()
75 +
76 +
77 +@pytest.fixture(scope="session")
78 +def client():
79 + client = openai.AsyncOpenAI(
80 + base_url="http://localhost:8000/v1",
81 + api_key="token-abc123",
82 + )
83 + yield client
84 +
85 +
86 +@pytest.mark.asyncio
87 +async def test_check_models(server, client: openai.AsyncOpenAI):
88 + models = await client.models.list()
89 + models = models.data
90 + served_model = models[0]
91 + assert served_model.id == MODEL_NAME
92 + assert all(model.root == MODEL_NAME for model in models)
93 +
94 +
95 +@pytest.mark.asyncio
96 +async def test_single_completion(server, client: openai.AsyncOpenAI):
97 + completion = await client.completions.create(model=MODEL_NAME,
98 + prompt="Hello, my name is",
99 + max_tokens=5,
100 + temperature=0.0)
101 +
102 + assert completion.id is not None
103 + assert completion.choices is not None and len(completion.choices) == 1
104 + assert completion.choices[0].text is not None and len(
105 + completion.choices[0].text) >= 5
106 + assert completion.choices[0].finish_reason == "length"
107 + assert completion.usage == openai.types.CompletionUsage(
108 + completion_tokens=5, prompt_tokens=6, total_tokens=11)
109 +
110 + # test using token IDs
111 + completion = await client.completions.create(
112 + model=MODEL_NAME,
113 + prompt=[0, 0, 0, 0, 0],
114 + max_tokens=5,
115 + temperature=0.0,
116 + )
117 + assert completion.choices[0].text is not None and len(
118 + completion.choices[0].text) >= 5
119 +
120 +
121 +@pytest.mark.asyncio
122 +async def test_single_chat_session(server, client: openai.AsyncOpenAI):
123 + messages = [{
124 + "role": "system",
125 + "content": "you are a helpful assistant"
126 + }, {
127 + "role": "user",
128 + "content": "what is 1+1?"
129 + }]
130 +
131 + # test single completion
132 + chat_completion = await client.chat.completions.create(model=MODEL_NAME,
133 + messages=messages,
134 + max_tokens=10,
135 + logprobs=True,
136 + top_logprobs=5)
137 + assert chat_completion.id is not None
138 + assert chat_completion.choices is not None and len(
139 + chat_completion.choices) == 1
140 + assert chat_completion.choices[0].message is not None
141 + assert chat_completion.choices[0].logprobs is not None
142 + assert chat_completion.choices[0].logprobs.top_logprobs is not None
143 + assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
144 + message = chat_completion.choices[0].message
145 + assert message.content is not None and len(message.content) >= 10
146 + assert message.role == "assistant"
147 + messages.append({"role": "assistant", "content": message.content})
148 +
149 + # test multi-turn dialogue
150 + messages.append({"role": "user", "content": "express your result in json"})
151 + chat_completion = await client.chat.completions.create(
152 + model=MODEL_NAME,
153 + messages=messages,
154 + max_tokens=10,
155 + )
156 + message = chat_completion.choices[0].message
157 + assert message.content is not None and len(message.content) >= 0
vllm/engine/async_llm_engine.py
+9 -1
@@ -7,7 +7,7 @@ from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
7
8 from transformers import PreTrainedTokenizer
9
10 -from vllm.config import ModelConfig
10 +from vllm.config import DecodingConfig, ModelConfig
11 from vllm.engine.arg_utils import AsyncEngineArgs
12 from vllm.engine.llm_engine import LLMEngine
13 from vllm.executor.ray_utils import initialize_ray_cluster, ray
@@ -697,6 +697,14 @@ class AsyncLLMEngine:
697 else:
698 return self.engine.get_model_config()
699
700 + async def get_decoding_config(self) -> DecodingConfig:
701 + """Get the decoding configuration of the vLLM engine."""
702 + if self.engine_use_ray:
703 + return await self.engine.get_decoding_config.remote( # type: ignore
704 + )
705 + else:
706 + return self.engine.get_decoding_config()
707 +
708 async def do_log_stats(self) -> None:
709 if self.engine_use_ray:
710 await self.engine.do_log_stats.remote() # type: ignore
vllm/engine/llm_engine.py
+4
@@ -467,6 +467,10 @@ class LLMEngine:
467 """Gets the model configuration."""
468 return self.model_config
469
470 + def get_decoding_config(self) -> DecodingConfig:
471 + """Gets the decoding configuration."""
472 + return self.decoding_config
473 +
474 def get_num_unfinished_requests(self) -> int:
475 """Gets the number of unfinished requests."""
476 return self.scheduler.get_num_unfinished_seq_groups()
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -101,7 +101,7 @@ class OpenAIServingChat(OpenAIServing):
101 request, prompt=prompt)
102 sampling_params = request.to_sampling_params()
103 lora_request = self._maybe_get_lora(request)
104 - decoding_config = self.engine.engine.decoding_config
104 + decoding_config = await self.engine.get_decoding_config()
105 guided_decoding_backend = request.guided_decoding_backend \
106 or decoding_config.guided_decoding_backend
107 guided_decode_logits_processor = (
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -89,7 +89,7 @@ class OpenAIServingCompletion(OpenAIServing):
89 try:
90 sampling_params = request.to_sampling_params()
91 lora_request = self._maybe_get_lora(request)
92 - decoding_config = self.engine.engine.decoding_config
92 + decoding_config = await self.engine.get_decoding_config()
93 guided_decoding_backend = request.guided_decoding_backend \
94 or decoding_config.guided_decoding_backend
95 guided_decode_logit_processor = (