| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import asyncio |
| 4 | import threading |
| 5 | |
| 6 | from vllm.engine.arg_utils import AsyncEngineArgs |
| 7 | from vllm.engine.async_llm_engine import AsyncLLMEngine |
| 8 | from vllm.outputs import RequestOutput |
| 9 | from vllm.sampling_params import SamplingParams |
| 10 | from vllm.v1.metrics.loggers import AggregatedLoggingStatLogger |
| 11 | |
| 12 | """ |
| 13 | To run this example, run the following commands simultaneously with |
| 14 | different CUDA_VISIBLE_DEVICES: |
| 15 | python examples/features/data_parallel/multi_instance_data_parallel.py |
| 16 | |
| 17 | vllm serve ibm-research/PowerMoE-3b -dp 2 -dpr 1 \ |
| 18 | --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 62300 \ |
| 19 | --data-parallel-size-local 1 --enforce-eager --headless |
| 20 | |
| 21 | Once both instances have completed the handshake, this example will |
| 22 | send a request to the instance with DP rank 1. |
| 23 | """ |
| 24 | |
| 25 | |
| 26 | def _do_background_logging(engine, interval, stop_event): |
| 27 | try: |
| 28 | while not stop_event.is_set(): |
| 29 | asyncio.run(engine.do_log_stats()) |
| 30 | stop_event.wait(interval) |
| 31 | except Exception as e: |
| 32 | print(f"vLLM background logging shutdown: {e}") |
| 33 | pass |
| 34 | |
| 35 | |
| 36 | async def main(): |
| 37 | engine_args = AsyncEngineArgs( |
| 38 | model="ibm-research/PowerMoE-3b", |
| 39 | data_parallel_size=2, |
| 40 | tensor_parallel_size=1, |
| 41 | dtype="auto", |
| 42 | max_model_len=2048, |
| 43 | data_parallel_address="127.0.0.1", |
| 44 | data_parallel_rpc_port=62300, |
| 45 | data_parallel_size_local=1, |
| 46 | enforce_eager=True, |
| 47 | enable_log_requests=True, |
| 48 | disable_custom_all_reduce=True, |
| 49 | ) |
| 50 | |
| 51 | engine_client = AsyncLLMEngine.from_engine_args( |
| 52 | engine_args, |
| 53 | # Example: Using aggregated logger |
| 54 | stat_loggers=[AggregatedLoggingStatLogger], |
| 55 | ) |
| 56 | stop_logging_event = threading.Event() |
| 57 | logging_thread = threading.Thread( |
| 58 | target=_do_background_logging, |
| 59 | args=(engine_client, 5, stop_logging_event), |
| 60 | daemon=True, |
| 61 | ) |
| 62 | logging_thread.start() |
| 63 | sampling_params = SamplingParams( |
| 64 | temperature=0.7, |
| 65 | top_p=0.9, |
| 66 | max_tokens=100, |
| 67 | ) |
| 68 | num_prompts = 10 |
| 69 | for i in range(num_prompts): |
| 70 | prompt = "Who won the 2004 World Series?" |
| 71 | final_output: RequestOutput | None = None |
| 72 | async for output in engine_client.generate( |
| 73 | prompt=prompt, |
| 74 | sampling_params=sampling_params, |
| 75 | request_id=f"abcdef-{i}", |
| 76 | data_parallel_rank=1, |
| 77 | ): |
| 78 | final_output = output |
| 79 | if final_output: |
| 80 | print(final_output.outputs[0].text) |
| 81 | |
| 82 | stop_logging_event.set() |
| 83 | logging_thread.join() |
| 84 | |
| 85 | |
| 86 | if __name__ == "__main__": |
| 87 | asyncio.run(main()) |