A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[bugfix] add seed in torchrun_example.py (#15980)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Apr 3, 2025 at 12:25 UTC
8b664706aa72315e436097d19257536228e01792
2 files changed
+10
examples/offline_inference/torchrun_example.py
+4
@@ -23,10 +23,14 @@ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
23
24
# Use `distributed_executor_backend="external_launcher"` so that
25
# this llm engine/instance only creates one worker.
26
+# it is important to set an explicit seed to make sure that
27
+# all ranks have the same random seed, so that sampling can be
28
+# deterministic across ranks.
29
llm = LLM(
30
model="facebook/opt-125m",
31
tensor_parallel_size=2,
32
distributed_executor_backend="external_launcher",
33
+ seed=0,
34
)
35
36
outputs = llm.generate(prompts, sampling_params)
vllm/config.py
+6
@@ -761,6 +761,12 @@ class ModelConfig:
761
self,
762
parallel_config: "ParallelConfig",
763
) -> None:
764
+
765
+ if parallel_config.distributed_executor_backend == "external_launcher":
766
+ assert self.seed is not None, (
767
+ "Seed must be set when using external launcher backend to "
768
+ "make sure sampling results are the same across workers.")
769
+
770
total_num_attention_heads = getattr(self.hf_text_config,
771
"num_attention_heads", 0)
772
tensor_parallel_size = parallel_config.tensor_parallel_size