A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Doc] Specify async engine args in docs (#9726)
Cyrus Leung committed
Oct 29, 2024 at 13:07 UTC
e74f2d448c9b984f6b2c91137c58919441456503
1 file changed
+2
vllm/engine/async_llm_engine.py
+2
@@ -999,6 +999,7 @@ class AsyncLLMEngine(EngineClient):
999
>>> # the complete example.
1000
>>>
1001
>>> # initialize the engine and the example input
1002
+ >>> # note that engine_args here is AsyncEngineArgs instance
1003
>>> engine = AsyncLLMEngine.from_engine_args(engine_args)
1004
>>> example_input = {
1005
>>> "prompt": "What is LLM?",
@@ -1082,6 +1083,7 @@ class AsyncLLMEngine(EngineClient):
1083
>>> # the complete example.
1084
>>>
1085
>>> # initialize the engine and the example input
1086
+ >>> # note that engine_args here is AsyncEngineArgs instance
1087
>>> engine = AsyncLLMEngine.from_engine_args(engine_args)
1088
>>> example_input = {
1089
>>> "input": "What is LLM?",