A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Quality] Add CI for formatting (#343)
Zhuohan Li committed
Jul 3, 2023 at 14:50 UTC
42e0c1df789a2079eeb219fa790cbf5678af662f
6 files changed
+113
-31
.github/workflows/pylint.yml
new
+31
@@ -0,0 +1,31 @@
1
+name: pylint
2
+
3
+on:
4
+ # Trigger the workflow on push or pull request,
5
+ # but only for the main branch
6
+ push:
7
+ branches:
8
+ - main
9
+ pull_request:
10
+ branches:
11
+ - main
12
+
13
+jobs:
14
+ pylint:
15
+ runs-on: ubuntu-latest
16
+ strategy:
17
+ matrix:
18
+ python-version: ["3.10"]
19
+ steps:
20
+ - uses: actions/checkout@v2
21
+ - name: Set up Python ${{ matrix.python-version }}
22
+ uses: actions/setup-python@v2
23
+ with:
24
+ python-version: ${{ matrix.python-version }}
25
+ - name: Install dependencies
26
+ run: |
27
+ python -m pip install --upgrade pip
28
+ pip install pylint==2.8.2
29
+ - name: Analysing the code with pylint
30
+ run: |
31
+ pylint vllm
.github/workflows/yapf.yml
new
+31
@@ -0,0 +1,31 @@
1
+name: yapf
2
+
3
+on:
4
+ # Trigger the workflow on push or pull request,
5
+ # but only for the main branch
6
+ push:
7
+ branches:
8
+ - main
9
+ pull_request:
10
+ branches:
11
+ - main
12
+jobs:
13
+ yapf:
14
+ runs-on: ubuntu-latest
15
+ strategy:
16
+ matrix:
17
+ python-version: ["3.10"]
18
+ steps:
19
+ - uses: actions/checkout@v2
20
+ - name: Set up Python ${{ matrix.python-version }}
21
+ uses: actions/setup-python@v2
22
+ with:
23
+ python-version: ${{ matrix.python-version }}
24
+ - name: Install dependencies
25
+ run: |
26
+ python -m pip install --upgrade pip
27
+ pip install yapf==0.32.0
28
+ pip install toml==0.10.2
29
+ - name: Running yapf
30
+ run: |
31
+ yapf --diff --recursive vllm --exclude 'vllm/model_executor/parallel_utils/**'
vllm/engine/async_llm_engine.py
+8
@@ -2,6 +2,7 @@ import asyncio
2
import time
3
from typing import Dict, List, Optional
4
5
+from vllm.config import ModelConfig
6
from vllm.engine.arg_utils import AsyncEngineArgs
7
from vllm.engine.llm_engine import LLMEngine
8
from vllm.engine.ray_utils import initialize_cluster, ray
@@ -206,6 +207,13 @@ class AsyncLLMEngine:
207
self.is_engine_running = False
208
self.kicking_request_id = None
209
210
+ async def get_model_config(self) -> ModelConfig:
211
+ """Get the model configuration of the vLLM engine."""
212
+ if self.engine_use_ray:
213
+ return await self.engine.get_model_config.remote()
214
+ else:
215
+ return self.engine.get_model_config()
216
+
217
@classmethod
218
def from_engine_args(cls,
219
engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
vllm/engine/llm_engine.py
+4
@@ -210,6 +210,10 @@ class LLMEngine:
210
"""
211
self.scheduler.abort_seq_group(request_id)
212
213
+ def get_model_config(self) -> ModelConfig:
214
+ """Gets the model configuration."""
215
+ return self.model_config
216
+
217
def get_num_unfinished_requests(self) -> int:
218
"""Gets the number of unfinished requests."""
219
return self.scheduler.get_num_unfinished_seq_groups()
vllm/entrypoints/openai/api_server.py
+37
-30
@@ -2,16 +2,19 @@
2
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
3
4
import argparse
5
+import asyncio
6
from http import HTTPStatus
7
import json
8
import time
8
-from typing import AsyncGenerator, Dict, List, Optional, Union, Any
9
+from typing import AsyncGenerator, Dict, List, Optional
10
11
import fastapi
12
from fastapi import BackgroundTasks, Request
13
from fastapi.exceptions import RequestValidationError
14
from fastapi.middleware.cors import CORSMiddleware
15
from fastapi.responses import JSONResponse, StreamingResponse
16
+from fastchat.conversation import (Conversation, SeparatorStyle,
17
+ get_conv_template)
18
import uvicorn
19
20
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -19,11 +22,10 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
22
from vllm.entrypoints.openai.protocol import (
23
CompletionRequest, CompletionResponse, CompletionResponseChoice,
24
CompletionResponseStreamChoice, CompletionStreamResponse,
22
- ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice,
23
- ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse,
24
- ChatMessage, DeltaMessage, ErrorResponse, LogProbs,
25
- ModelCard, ModelList, ModelPermission, UsageInfo)
26
-from fastchat.conversation import Conversation, SeparatorStyle, get_conv_template
25
+ ChatCompletionRequest, ChatCompletionResponse,
26
+ ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
27
+ ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
28
+ LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
29
from vllm.logger import init_logger
30
from vllm.outputs import RequestOutput
31
from vllm.sampling_params import SamplingParams
@@ -95,15 +97,15 @@ async def get_gen_prompt(request) -> str:
97
return prompt
98
99
98
-async def check_length(request, prompt, engine):
99
- if hasattr(engine.engine.model_config.hf_config, "max_sequence_length"):
100
- context_len = engine.engine.model_config.hf_config.max_sequence_length
101
- elif hasattr(engine.engine.model_config.hf_config, "seq_length"):
102
- context_len = engine.engine.model_config.hf_config.seq_length
103
- elif hasattr(engine.engine.model_config.hf_config, "max_position_embeddings"):
104
- context_len = engine.engine.model_config.hf_config.max_position_embeddings
105
- elif hasattr(engine.engine.model_config.hf_config, "seq_length"):
106
- context_len = engine.engine.model_config.hf_config.seq_length
100
+async def check_length(request, prompt, model_config):
101
+ if hasattr(model_config.hf_config, "max_sequence_length"):
102
+ context_len = model_config.hf_config.max_sequence_length
103
+ elif hasattr(model_config.hf_config, "seq_length"):
104
+ context_len = model_config.hf_config.seq_length
105
+ elif hasattr(model_config.hf_config, "max_position_embeddings"):
106
+ context_len = model_config.hf_config.max_position_embeddings
107
+ elif hasattr(model_config.hf_config, "seq_length"):
108
+ context_len = model_config.hf_config.seq_length
109
else:
110
context_len = 2048
111
@@ -182,7 +184,7 @@ async def create_chat_completion(raw_request: Request):
184
"logit_bias is not currently supported")
185
186
prompt = await get_gen_prompt(request)
185
- error_check_ret = await check_length(request, prompt, engine)
187
+ error_check_ret = await check_length(request, prompt, engine_model_config)
188
if error_check_ret is not None:
189
return error_check_ret
190
@@ -206,15 +208,16 @@ async def create_chat_completion(raw_request: Request):
208
except ValueError as e:
209
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
210
209
- result_generator = engine.generate(prompt, sampling_params,
210
- request_id)
211
+ result_generator = engine.generate(prompt, sampling_params, request_id)
212
213
async def abort_request() -> None:
214
await engine.abort(request_id)
215
215
- def create_stream_response_json(index: int,
216
- text: str,
217
- finish_reason: Optional[str] = None) -> str:
216
+ def create_stream_response_json(
217
+ index: int,
218
+ text: str,
219
+ finish_reason: Optional[str] = None,
220
+ ) -> str:
221
choice_data = ChatCompletionResponseStreamChoice(
222
index=index,
223
delta=DeltaMessage(content=text),
@@ -238,10 +241,11 @@ async def create_chat_completion(raw_request: Request):
241
delta=DeltaMessage(role="assistant"),
242
finish_reason=None,
243
)
241
- chunk = ChatCompletionStreamResponse(
242
- id=request_id, choices=[choice_data], model=model_name
243
- )
244
- yield f"data: {chunk.json(exclude_unset=True, ensure_ascii=False)}\n\n"
244
+ chunk = ChatCompletionStreamResponse(id=request_id,
245
+ choices=[choice_data],
246
+ model=model_name)
247
+ data = chunk.json(exclude_unset=True, ensure_ascii=False)
248
+ yield f"data: {data}\n\n"
249
250
previous_texts = [""] * request.n
251
previous_num_tokens = [0] * request.n
@@ -295,8 +299,8 @@ async def create_chat_completion(raw_request: Request):
299
choices.append(choice_data)
300
301
num_prompt_tokens = len(final_res.prompt_token_ids)
298
- num_generated_tokens = sum(len(output.token_ids)
299
- for output in final_res.outputs)
302
+ num_generated_tokens = sum(
303
+ len(output.token_ids) for output in final_res.outputs)
304
usage = UsageInfo(
305
prompt_tokens=num_prompt_tokens,
306
completion_tokens=num_generated_tokens,
@@ -314,9 +318,11 @@ async def create_chat_completion(raw_request: Request):
318
# When user requests streaming but we don't stream, we still need to
319
# return a streaming response with a single event.
320
response_json = response.json(ensure_ascii=False)
321
+
322
async def fake_stream_generator() -> AsyncGenerator[str, None]:
323
yield f"data: {response_json}\n\n"
324
yield "data: [DONE]\n\n"
325
+
326
return StreamingResponse(fake_stream_generator(),
327
media_type="text/event-stream")
328
@@ -367,9 +373,9 @@ async def create_completion(raw_request: Request):
373
return create_error_response(HTTPStatus.BAD_REQUEST,
374
"please provide at least one prompt")
375
if len(request.prompt) > 1:
370
- return create_error_response(HTTPStatus.BAD_REQUEST,
371
- "multiple prompts in a batch is not "
372
- "currently supported")
376
+ return create_error_response(
377
+ HTTPStatus.BAD_REQUEST,
378
+ "multiple prompts in a batch is not currently supported")
379
prompt = request.prompt[0]
380
else:
381
prompt = request.prompt
@@ -571,6 +577,7 @@ if __name__ == "__main__":
577
578
engine_args = AsyncEngineArgs.from_cli_args(args)
579
engine = AsyncLLMEngine.from_engine_args(engine_args)
580
+ engine_model_config = asyncio.run(engine.get_model_config())
581
582
# A separate tokenizer to map token IDs to strings.
583
tokenizer = get_tokenizer(engine_args.tokenizer,
vllm/model_executor/models/bloom.py
+2
-1
@@ -1,5 +1,6 @@
1
# coding=utf-8
2
-# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
4
# Copyright 2023 The CacheFlow team.
5
# Copyright 2022 HuggingFace Inc. team and BigScience workshop.
6
#