@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e74f2d448c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Doc] Specify async engine args in docs (#9726)

Cyrus Leung committed Oct 29, 2024 at 13:07 UTC e74f2d448c9b984f6b2c91137c58919441456503
1 file changed +2
vllm/engine/async_llm_engine.py
+2
@@ -999,6 +999,7 @@ class AsyncLLMEngine(EngineClient):
999 >>> # the complete example.
1000 >>>
1001 >>> # initialize the engine and the example input
1002 + >>> # note that engine_args here is AsyncEngineArgs instance
1003 >>> engine = AsyncLLMEngine.from_engine_args(engine_args)
1004 >>> example_input = {
1005 >>> "prompt": "What is LLM?",
@@ -1082,6 +1083,7 @@ class AsyncLLMEngine(EngineClient):
1083 >>> # the complete example.
1084 >>>
1085 >>> # initialize the engine and the example input
1086 + >>> # note that engine_args here is AsyncEngineArgs instance
1087 >>> engine = AsyncLLMEngine.from_engine_args(engine_args)
1088 >>> example_input = {
1089 >>> "input": "What is LLM?",