@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1666e66443

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

Add "/server_info" endpoint in api_server to retrieve the vllm_config. (#16572)

Signed-off-by: Xihui Cang <xihuicang@gmail.com>

Xihui Cang committed Apr 15, 2025 at 19:50 UTC 1666e664435a0b918b7186a020bcc59b8b3216c0

7 files changed +37 -6

vllm/engine/async_llm_engine.py

@@ -1167,6 +1167,10 @@ class AsyncLLMEngine(EngineClient):

1167 exception=asyncio.CancelledError,

1168 verbose=self.log_requests)

1169

1170 + async def get_vllm_config(self) -> VllmConfig:

1171 + """Get the vllm configuration of the vLLM engine."""

1172 + return self.engine.get_vllm_config()

1173 +

1174 async def get_model_config(self) -> ModelConfig:

1175 """Get the model configuration of the vLLM engine."""

1176 return self.engine.get_model_config()

vllm/engine/llm_engine.py

@@ -914,6 +914,10 @@ class LLMEngine:

914 scheduler.abort_seq_group(

915 request_id, seq_id_to_seq_group=self.seq_id_to_seq_group)

916

917 + def get_vllm_config(self) -> VllmConfig:

918 + """Gets the vllm configuration."""

919 + return self.vllm_config

920 +

921 def get_model_config(self) -> ModelConfig:

922 """Gets the model configuration."""

923 return self.model_config

vllm/engine/multiprocessing/client.py

@@ -93,6 +93,7 @@ class MQLLMEngineClient(EngineClient):

93 self._errored_with: Optional[BaseException] = None

95 # Get the configs.

96 + self.vllm_config = engine_config

97 self.model_config = engine_config.model_config

98 self.decoding_config = engine_config.decoding_config

@@ -377,6 +378,9 @@ class MQLLMEngineClient(EngineClient):

378 async def get_tokenizer(self, lora_request: Optional[LoRARequest] = None):

379 return await self.tokenizer.get_lora_tokenizer_async(lora_request)

380

381 + async def get_vllm_config(self) -> VllmConfig:

382 + return self.vllm_config

383 +

384 async def get_decoding_config(self) -> DecodingConfig:

385 return self.decoding_config

386

vllm/engine/protocol.py

+6 -1

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod

5 from typing import AsyncGenerator, List, Mapping, Optional

7 from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function

8 -from vllm.config import DecodingConfig, ModelConfig

8 +from vllm.config import DecodingConfig, ModelConfig, VllmConfig

9 from vllm.core.scheduler import SchedulerOutputs

10 from vllm.inputs.data import PromptType, TokensPrompt

11 from vllm.inputs.parse import is_explicit_encoder_decoder_prompt

@@ -220,6 +220,11 @@ class EngineClient(ABC):

220 """

221 ...

222

223 + @abstractmethod

224 + async def get_vllm_config(self) -> VllmConfig:

225 + """Get the vllm configuration of the vLLM engine."""

226 + ...

227 +

228 @abstractmethod

229 async def get_model_config(self) -> ModelConfig:

230 """Get the model configuration of the vLLM engine."""

vllm/entrypoints/openai/api_server.py

+12 -4

@@ -30,7 +30,7 @@ from starlette.routing import Mount

30 from typing_extensions import assert_never

32 import vllm.envs as envs

33 -from vllm.config import ModelConfig

33 +from vllm.config import VllmConfig

34 from vllm.engine.arg_utils import AsyncEngineArgs

35 from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore

36 from vllm.engine.multiprocessing.client import MQLLMEngineClient

@@ -327,6 +327,7 @@ def mount_metrics(app: FastAPI):

327 "/load",

328 "/ping",

329 "/version",

330 + "/server_info",

331 ],

332 registry=registry,

333 ).add().instrument(app).expose(app)

@@ -687,6 +688,11 @@ TASK_HANDLERS: dict[str, dict[str, tuple]] = {

688

689 if envs.VLLM_SERVER_DEV_MODE:

690

691 + @router.get("/server_info")

692 + async def show_server_info(raw_request: Request):

693 + server_info = {"vllm_config": str(raw_request.app.state.vllm_config)}

694 + return JSONResponse(content=server_info)

695 +

696 @router.post("/reset_prefix_cache")

697 async def reset_prefix_cache(raw_request: Request):

698 """

@@ -894,7 +900,7 @@ def build_app(args: Namespace) -> FastAPI:

900

901 async def init_app_state(

902 engine_client: EngineClient,

897 - model_config: ModelConfig,

903 + vllm_config: VllmConfig,

904 state: State,

905 args: Namespace,

906 ) -> None:

@@ -915,6 +921,8 @@ async def init_app_state(

921

922 state.engine_client = engine_client

923 state.log_stats = not args.disable_log_stats

924 + state.vllm_config = vllm_config

925 + model_config = vllm_config.model_config

926

927 resolved_chat_template = load_chat_template(args.chat_template)

928 if resolved_chat_template is not None:

@@ -1069,8 +1077,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:

1077 async with build_async_engine_client(args) as engine_client:

1078 app = build_app(args)

1079

1072 - model_config = await engine_client.get_model_config()

1073 - await init_app_state(engine_client, model_config, app.state, args)

1080 + vllm_config = await engine_client.get_vllm_config()

1081 + await init_app_state(engine_client, vllm_config, app.state, args)

1082

1083 def _listen_addr(a: str) -> str:

1084 if is_valid_ipv6_address(a):

vllm/v1/engine/async_llm.py

+4 -1

@@ -64,7 +64,7 @@ class AsyncLLM(EngineClient):

64 assert start_engine_loop

66 self.model_config = vllm_config.model_config

67 -

67 + self.vllm_config = vllm_config

68 self.log_requests = log_requests

69 self.log_stats = log_stats

@@ -379,6 +379,9 @@ class AsyncLLM(EngineClient):

379 ):

380 raise ValueError("Not Supported on V1 yet.")

381

382 + async def get_vllm_config(self) -> VllmConfig:

383 + return self.vllm_config

384 +

385 async def get_model_config(self) -> ModelConfig:

386 return self.model_config

387

vllm/v1/engine/llm_engine.py

@@ -230,6 +230,9 @@ class LLMEngine:

230

231 return processed_outputs.request_outputs

232

233 + def get_vllm_config(self):

234 + return self.vllm_config

235 +

236 def get_model_config(self):

237 return self.model_config

238