A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Misc] Set default value of seed to None (#14274)
Signed-off-by: மனோஜ்குமார் பழனிச்சாமி <smartmanoj42857@gmail.com>
மனோஜ்குமார் பழனிச்சாமி committed
Mar 7, 2025 at 16:10 UTC
cc10281498fc2a6eb804274dcf22e6cb766f7aa7
9 files changed
+15
-7
tests/distributed/test_torchrun_example.py
+2
-1
@@ -25,7 +25,8 @@ llm = LLM(model="facebook/opt-125m",
25
tensor_parallel_size=2,
26
distributed_executor_backend="external_launcher",
27
gpu_memory_utilization=random.uniform(0.7, 0.9),
28
- swap_space=random.randint(1, 4))
28
+ swap_space=random.randint(1, 4),
29
+ seed=0)
30
31
outputs = llm.generate(prompts, sampling_params)
32
tests/entrypoints/llm/test_encode.py
+2
-1
@@ -34,7 +34,8 @@ def llm():
34
max_num_batched_tokens=32768,
35
tensor_parallel_size=1,
36
gpu_memory_utilization=0.75,
37
- enforce_eager=True)
37
+ enforce_eager=True,
38
+ seed=0)
39
40
with llm.deprecate_legacy_api():
41
yield weakref.proxy(llm)
tests/entrypoints/llm/test_guided_generate.py
+1
-1
@@ -21,7 +21,7 @@ GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
21
def llm():
22
# pytest caches the fixture so we use weakref.proxy to
23
# enable garbage collection
24
- llm = LLM(model=MODEL_NAME, max_model_len=1024)
24
+ llm = LLM(model=MODEL_NAME, max_model_len=1024, seed=0)
25
26
with llm.deprecate_legacy_api():
27
yield weakref.proxy(llm)
tests/entrypoints/openai/test_chat_echo.py
+2
@@ -24,6 +24,8 @@ def server():
24
"4080",
25
"--chat-template",
26
DUMMY_CHAT_TEMPLATE,
27
+ "--seed",
28
+ "0",
29
]
30
31
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
tests/entrypoints/openai/test_metrics.py
+2
@@ -47,6 +47,8 @@ def default_server_args():
47
"--enforce-eager",
48
"--max-num-seqs",
49
"128",
50
+ "--seed",
51
+ "0",
52
]
53
54
tests/entrypoints/openai/test_root_path.py
+2
@@ -30,6 +30,8 @@ def server():
30
"/" + ROOT_PATH,
31
"--chat-template",
32
DUMMY_CHAT_TEMPLATE,
33
+ "--seed",
34
+ "0",
35
]
36
envs = os.environ.copy()
37
vllm/engine/arg_utils.py
+1
-1
@@ -104,7 +104,7 @@ class EngineArgs:
104
config_format: ConfigFormat = ConfigFormat.AUTO
105
dtype: str = 'auto'
106
kv_cache_dtype: str = 'auto'
107
- seed: int = 0
107
+ seed: Optional[int] = None
108
max_model_len: Optional[int] = None
109
# Note: Specifying a custom executor backend by passing a class
110
# is intended for expert use only. The API may change without
vllm/entrypoints/llm.py
+1
-1
@@ -169,7 +169,7 @@ class LLM:
169
quantization: Optional[str] = None,
170
revision: Optional[str] = None,
171
tokenizer_revision: Optional[str] = None,
172
- seed: int = 0,
172
+ seed: Optional[int] = None,
173
gpu_memory_utilization: float = 0.9,
174
swap_space: float = 4,
175
cpu_offload_gb: float = 0,
vllm/utils.py
+2
-2
@@ -644,7 +644,7 @@ def create_kv_caches_with_random_flash(
644
head_size: int,
645
cache_dtype: Optional[Union[str, torch.dtype]],
646
model_dtype: Optional[Union[str, torch.dtype]] = None,
647
- seed: int = 0,
647
+ seed: Optional[int] = None,
648
device: Optional[str] = "cuda",
649
) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
650
from vllm.platforms import current_platform
@@ -681,7 +681,7 @@ def create_kv_caches_with_random(
681
head_size: int,
682
cache_dtype: Optional[Union[str, torch.dtype]],
683
model_dtype: Optional[Union[str, torch.dtype]] = None,
684
- seed: int = 0,
684
+ seed: Optional[int] = None,
685
device: Optional[str] = "cuda",
686
) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
687