A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add vLLM version info to logs and openai API server (#3161)
Jason Cox committed
Mar 3, 2024 at 00:00 UTC
d65fac2738f0287a41955b45df76a2d5a919bff6
2 files changed
+10
-1
vllm/engine/llm_engine.py
+2
-1
@@ -7,6 +7,7 @@ import importlib
7
from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8
Union)
9
10
+import vllm
11
from vllm.lora.request import LoRARequest
12
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
13
ParallelConfig, SchedulerConfig, LoRAConfig)
@@ -85,7 +86,7 @@ class LLMEngine:
86
log_stats: bool,
87
) -> None:
88
logger.info(
88
- "Initializing an LLM engine with config: "
89
+ f"Initializing an LLM engine (v{vllm.__version__}) with config: "
90
f"model={model_config.model!r}, "
91
f"tokenizer={model_config.tokenizer!r}, "
92
f"tokenizer_mode={model_config.tokenizer_mode}, "
vllm/entrypoints/openai/api_server.py
+8
@@ -15,6 +15,7 @@ from fastapi.exceptions import RequestValidationError
15
from fastapi.middleware.cors import CORSMiddleware
16
from fastapi.responses import JSONResponse, StreamingResponse, Response
17
18
+import vllm
19
from vllm.engine.arg_utils import AsyncEngineArgs
20
from vllm.engine.async_llm_engine import AsyncLLMEngine
21
from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse
@@ -168,6 +169,12 @@ async def show_available_models():
169
return JSONResponse(content=models.model_dump())
170
171
172
+@app.get("/version")
173
+async def show_version():
174
+ ver = {"version": vllm.__version__}
175
+ return JSONResponse(content=ver)
176
+
177
+
178
@app.post("/v1/chat/completions")
179
async def create_chat_completion(request: ChatCompletionRequest,
180
raw_request: Request):
@@ -231,6 +238,7 @@ if __name__ == "__main__":
238
f"Invalid middleware {middleware}. Must be a function or a class."
239
)
240
241
+ logger.info(f"vLLM API server version {vllm.__version__}")
242
logger.info(f"args: {args}")
243
244
if args.served_model_name is not None: