@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d65fac2738
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add vLLM version info to logs and openai API server (#3161)

Jason Cox committed Mar 3, 2024 at 00:00 UTC d65fac2738f0287a41955b45df76a2d5a919bff6
2 files changed +10 -1
vllm/engine/llm_engine.py
+2 -1
@@ -7,6 +7,7 @@ import importlib
7 from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8 Union)
9
10 +import vllm
11 from vllm.lora.request import LoRARequest
12 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
13 ParallelConfig, SchedulerConfig, LoRAConfig)
@@ -85,7 +86,7 @@ class LLMEngine:
86 log_stats: bool,
87 ) -> None:
88 logger.info(
88 - "Initializing an LLM engine with config: "
89 + f"Initializing an LLM engine (v{vllm.__version__}) with config: "
90 f"model={model_config.model!r}, "
91 f"tokenizer={model_config.tokenizer!r}, "
92 f"tokenizer_mode={model_config.tokenizer_mode}, "
vllm/entrypoints/openai/api_server.py
+8
@@ -15,6 +15,7 @@ from fastapi.exceptions import RequestValidationError
15 from fastapi.middleware.cors import CORSMiddleware
16 from fastapi.responses import JSONResponse, StreamingResponse, Response
17
18 +import vllm
19 from vllm.engine.arg_utils import AsyncEngineArgs
20 from vllm.engine.async_llm_engine import AsyncLLMEngine
21 from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse
@@ -168,6 +169,12 @@ async def show_available_models():
169 return JSONResponse(content=models.model_dump())
170
171
172 +@app.get("/version")
173 +async def show_version():
174 + ver = {"version": vllm.__version__}
175 + return JSONResponse(content=ver)
176 +
177 +
178 @app.post("/v1/chat/completions")
179 async def create_chat_completion(request: ChatCompletionRequest,
180 raw_request: Request):
@@ -231,6 +238,7 @@ if __name__ == "__main__":
238 f"Invalid middleware {middleware}. Must be a function or a class."
239 )
240
241 + logger.info(f"vLLM API server version {vllm.__version__}")
242 logger.info(f"args: {args}")
243
244 if args.served_model_name is not None: