A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[CI/Build] Update defaults for test reproducibility (#14893)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 17, 2025 at 10:38 UTC
8a5a9b70d702feb17e79691870c638b0f1e71192
1 file changed
+16
-2
tests/conftest.py
+16
-2
@@ -681,6 +681,17 @@ def hf_runner():
681
682
683
class VllmRunner:
684
+ """
685
+ The default value of some arguments have been modified from
686
+ :class:`~vllm.LLM` as follows:
687
+ - `trust_remote_code`: Set to `True` instead of `False` for convenience.
688
+ - `seed`: Set to `0` instead of `None` for test reproducibility.
689
+ - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.
690
+ - `block_size`: Set to `16` instead of `None` to reduce memory usage.
691
+ - `enable_chunked_prefill`: Set to `False` instead of `None` for
692
+ test reproducibility.
693
+ - `enforce_eager`: Set to `False` instead of `None` to test CUDA graph.
694
+ """
695
696
def __init__(
697
self,
@@ -688,6 +699,8 @@ class VllmRunner:
699
task: TaskOption = "auto",
700
tokenizer_name: Optional[str] = None,
701
tokenizer_mode: str = "auto",
702
+ trust_remote_code: bool = True,
703
+ seed: Optional[int] = 0,
704
# Use smaller max model length, otherwise bigger model cannot run due
705
# to kv cache size limit.
706
max_model_len: int = 1024,
@@ -695,7 +708,7 @@ class VllmRunner:
708
disable_log_stats: bool = True,
709
tensor_parallel_size: int = 1,
710
block_size: int = 16,
698
- enable_chunked_prefill: bool = False,
711
+ enable_chunked_prefill: Optional[bool] = False,
712
swap_space: int = 4,
713
enforce_eager: Optional[bool] = False,
714
**kwargs,
@@ -705,8 +718,9 @@ class VllmRunner:
718
task=task,
719
tokenizer=tokenizer_name,
720
tokenizer_mode=tokenizer_mode,
708
- trust_remote_code=True,
721
+ trust_remote_code=trust_remote_code,
722
dtype=dtype,
723
+ seed=seed,
724
swap_space=swap_space,
725
enforce_eager=enforce_eager,
726
disable_log_stats=disable_log_stats,