@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 42e0c1df78
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Quality] Add CI for formatting (#343)

Zhuohan Li committed Jul 3, 2023 at 14:50 UTC 42e0c1df789a2079eeb219fa790cbf5678af662f
6 files changed +113 -31
.github/workflows/pylint.yml new
+31
@@ -0,0 +1,31 @@
1 +name: pylint
2 +
3 +on:
4 + # Trigger the workflow on push or pull request,
5 + # but only for the main branch
6 + push:
7 + branches:
8 + - main
9 + pull_request:
10 + branches:
11 + - main
12 +
13 +jobs:
14 + pylint:
15 + runs-on: ubuntu-latest
16 + strategy:
17 + matrix:
18 + python-version: ["3.10"]
19 + steps:
20 + - uses: actions/checkout@v2
21 + - name: Set up Python ${{ matrix.python-version }}
22 + uses: actions/setup-python@v2
23 + with:
24 + python-version: ${{ matrix.python-version }}
25 + - name: Install dependencies
26 + run: |
27 + python -m pip install --upgrade pip
28 + pip install pylint==2.8.2
29 + - name: Analysing the code with pylint
30 + run: |
31 + pylint vllm
.github/workflows/yapf.yml new
+31
@@ -0,0 +1,31 @@
1 +name: yapf
2 +
3 +on:
4 + # Trigger the workflow on push or pull request,
5 + # but only for the main branch
6 + push:
7 + branches:
8 + - main
9 + pull_request:
10 + branches:
11 + - main
12 +jobs:
13 + yapf:
14 + runs-on: ubuntu-latest
15 + strategy:
16 + matrix:
17 + python-version: ["3.10"]
18 + steps:
19 + - uses: actions/checkout@v2
20 + - name: Set up Python ${{ matrix.python-version }}
21 + uses: actions/setup-python@v2
22 + with:
23 + python-version: ${{ matrix.python-version }}
24 + - name: Install dependencies
25 + run: |
26 + python -m pip install --upgrade pip
27 + pip install yapf==0.32.0
28 + pip install toml==0.10.2
29 + - name: Running yapf
30 + run: |
31 + yapf --diff --recursive vllm --exclude 'vllm/model_executor/parallel_utils/**'
vllm/engine/async_llm_engine.py
+8
@@ -2,6 +2,7 @@ import asyncio
2 import time
3 from typing import Dict, List, Optional
4
5 +from vllm.config import ModelConfig
6 from vllm.engine.arg_utils import AsyncEngineArgs
7 from vllm.engine.llm_engine import LLMEngine
8 from vllm.engine.ray_utils import initialize_cluster, ray
@@ -206,6 +207,13 @@ class AsyncLLMEngine:
207 self.is_engine_running = False
208 self.kicking_request_id = None
209
210 + async def get_model_config(self) -> ModelConfig:
211 + """Get the model configuration of the vLLM engine."""
212 + if self.engine_use_ray:
213 + return await self.engine.get_model_config.remote()
214 + else:
215 + return self.engine.get_model_config()
216 +
217 @classmethod
218 def from_engine_args(cls,
219 engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
vllm/engine/llm_engine.py
+4
@@ -210,6 +210,10 @@ class LLMEngine:
210 """
211 self.scheduler.abort_seq_group(request_id)
212
213 + def get_model_config(self) -> ModelConfig:
214 + """Gets the model configuration."""
215 + return self.model_config
216 +
217 def get_num_unfinished_requests(self) -> int:
218 """Gets the number of unfinished requests."""
219 return self.scheduler.get_num_unfinished_seq_groups()
vllm/entrypoints/openai/api_server.py
+37 -30
@@ -2,16 +2,19 @@
2 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
3
4 import argparse
5 +import asyncio
6 from http import HTTPStatus
7 import json
8 import time
8 -from typing import AsyncGenerator, Dict, List, Optional, Union, Any
9 +from typing import AsyncGenerator, Dict, List, Optional
10
11 import fastapi
12 from fastapi import BackgroundTasks, Request
13 from fastapi.exceptions import RequestValidationError
14 from fastapi.middleware.cors import CORSMiddleware
15 from fastapi.responses import JSONResponse, StreamingResponse
16 +from fastchat.conversation import (Conversation, SeparatorStyle,
17 + get_conv_template)
18 import uvicorn
19
20 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -19,11 +22,10 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
22 from vllm.entrypoints.openai.protocol import (
23 CompletionRequest, CompletionResponse, CompletionResponseChoice,
24 CompletionResponseStreamChoice, CompletionStreamResponse,
22 - ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice,
23 - ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse,
24 - ChatMessage, DeltaMessage, ErrorResponse, LogProbs,
25 - ModelCard, ModelList, ModelPermission, UsageInfo)
26 -from fastchat.conversation import Conversation, SeparatorStyle, get_conv_template
25 + ChatCompletionRequest, ChatCompletionResponse,
26 + ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
27 + ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
28 + LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
29 from vllm.logger import init_logger
30 from vllm.outputs import RequestOutput
31 from vllm.sampling_params import SamplingParams
@@ -95,15 +97,15 @@ async def get_gen_prompt(request) -> str:
97 return prompt
98
99
98 -async def check_length(request, prompt, engine):
99 - if hasattr(engine.engine.model_config.hf_config, "max_sequence_length"):
100 - context_len = engine.engine.model_config.hf_config.max_sequence_length
101 - elif hasattr(engine.engine.model_config.hf_config, "seq_length"):
102 - context_len = engine.engine.model_config.hf_config.seq_length
103 - elif hasattr(engine.engine.model_config.hf_config, "max_position_embeddings"):
104 - context_len = engine.engine.model_config.hf_config.max_position_embeddings
105 - elif hasattr(engine.engine.model_config.hf_config, "seq_length"):
106 - context_len = engine.engine.model_config.hf_config.seq_length
100 +async def check_length(request, prompt, model_config):
101 + if hasattr(model_config.hf_config, "max_sequence_length"):
102 + context_len = model_config.hf_config.max_sequence_length
103 + elif hasattr(model_config.hf_config, "seq_length"):
104 + context_len = model_config.hf_config.seq_length
105 + elif hasattr(model_config.hf_config, "max_position_embeddings"):
106 + context_len = model_config.hf_config.max_position_embeddings
107 + elif hasattr(model_config.hf_config, "seq_length"):
108 + context_len = model_config.hf_config.seq_length
109 else:
110 context_len = 2048
111
@@ -182,7 +184,7 @@ async def create_chat_completion(raw_request: Request):
184 "logit_bias is not currently supported")
185
186 prompt = await get_gen_prompt(request)
185 - error_check_ret = await check_length(request, prompt, engine)
187 + error_check_ret = await check_length(request, prompt, engine_model_config)
188 if error_check_ret is not None:
189 return error_check_ret
190
@@ -206,15 +208,16 @@ async def create_chat_completion(raw_request: Request):
208 except ValueError as e:
209 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
210
209 - result_generator = engine.generate(prompt, sampling_params,
210 - request_id)
211 + result_generator = engine.generate(prompt, sampling_params, request_id)
212
213 async def abort_request() -> None:
214 await engine.abort(request_id)
215
215 - def create_stream_response_json(index: int,
216 - text: str,
217 - finish_reason: Optional[str] = None) -> str:
216 + def create_stream_response_json(
217 + index: int,
218 + text: str,
219 + finish_reason: Optional[str] = None,
220 + ) -> str:
221 choice_data = ChatCompletionResponseStreamChoice(
222 index=index,
223 delta=DeltaMessage(content=text),
@@ -238,10 +241,11 @@ async def create_chat_completion(raw_request: Request):
241 delta=DeltaMessage(role="assistant"),
242 finish_reason=None,
243 )
241 - chunk = ChatCompletionStreamResponse(
242 - id=request_id, choices=[choice_data], model=model_name
243 - )
244 - yield f"data: {chunk.json(exclude_unset=True, ensure_ascii=False)}\n\n"
244 + chunk = ChatCompletionStreamResponse(id=request_id,
245 + choices=[choice_data],
246 + model=model_name)
247 + data = chunk.json(exclude_unset=True, ensure_ascii=False)
248 + yield f"data: {data}\n\n"
249
250 previous_texts = [""] * request.n
251 previous_num_tokens = [0] * request.n
@@ -295,8 +299,8 @@ async def create_chat_completion(raw_request: Request):
299 choices.append(choice_data)
300
301 num_prompt_tokens = len(final_res.prompt_token_ids)
298 - num_generated_tokens = sum(len(output.token_ids)
299 - for output in final_res.outputs)
302 + num_generated_tokens = sum(
303 + len(output.token_ids) for output in final_res.outputs)
304 usage = UsageInfo(
305 prompt_tokens=num_prompt_tokens,
306 completion_tokens=num_generated_tokens,
@@ -314,9 +318,11 @@ async def create_chat_completion(raw_request: Request):
318 # When user requests streaming but we don't stream, we still need to
319 # return a streaming response with a single event.
320 response_json = response.json(ensure_ascii=False)
321 +
322 async def fake_stream_generator() -> AsyncGenerator[str, None]:
323 yield f"data: {response_json}\n\n"
324 yield "data: [DONE]\n\n"
325 +
326 return StreamingResponse(fake_stream_generator(),
327 media_type="text/event-stream")
328
@@ -367,9 +373,9 @@ async def create_completion(raw_request: Request):
373 return create_error_response(HTTPStatus.BAD_REQUEST,
374 "please provide at least one prompt")
375 if len(request.prompt) > 1:
370 - return create_error_response(HTTPStatus.BAD_REQUEST,
371 - "multiple prompts in a batch is not "
372 - "currently supported")
376 + return create_error_response(
377 + HTTPStatus.BAD_REQUEST,
378 + "multiple prompts in a batch is not currently supported")
379 prompt = request.prompt[0]
380 else:
381 prompt = request.prompt
@@ -571,6 +577,7 @@ if __name__ == "__main__":
577
578 engine_args = AsyncEngineArgs.from_cli_args(args)
579 engine = AsyncLLMEngine.from_engine_args(engine_args)
580 + engine_model_config = asyncio.run(engine.get_model_config())
581
582 # A separate tokenizer to map token IDs to strings.
583 tokenizer = get_tokenizer(engine_args.tokenizer,
vllm/model_executor/models/bloom.py
+2 -1
@@ -1,5 +1,6 @@
1 # coding=utf-8
2 -# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
4 # Copyright 2023 The CacheFlow team.
5 # Copyright 2022 HuggingFace Inc. team and BigScience workshop.
6 #