A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 08:45 UTC
[CI] Stabilize cpu offload compressed tensors test (#41102)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
May 4, 2026 at 00:22 UTC
6ec9bbec384b14401f901af189754f7d8a6754e2
2 files changed
+47
-25
tests/quantization/test_cpu_offload.py
+1
@@ -70,4 +70,5 @@ def test_cpu_offload_compressed_tensors(monkeypatch):
70
["--enforce_eager"],
71
["--enforce_eager", "--cpu-offload-gb", "1"],
72
max_wait_seconds=480,
73
+ include_seeded_sampling=False,
74
)
tests/utils.py
+46
-25
@@ -782,6 +782,7 @@ def _test_completion(
782
model: str,
783
prompt: str,
784
token_ids: list[int],
785
+ include_seeded_sampling: bool = True,
786
):
787
results = []
788
@@ -816,33 +817,40 @@ def _test_completion(
817
}
818
)
819
819
- # test seeded random sampling
820
- completion = client.completions.create(
821
- model=model, prompt=prompt, max_tokens=5, seed=33, temperature=1.0
822
- )
820
+ if include_seeded_sampling:
821
+ # test seeded random sampling
822
+ completion = client.completions.create(
823
+ model=model, prompt=prompt, max_tokens=5, seed=33, temperature=1.0
824
+ )
825
824
- results.append(
825
- {
826
- "test": "seeded_sampling",
827
- "text": completion.choices[0].text,
828
- "finish_reason": completion.choices[0].finish_reason,
829
- "usage": completion.usage,
830
- }
831
- )
826
+ results.append(
827
+ {
828
+ "test": "seeded_sampling",
829
+ "text": completion.choices[0].text,
830
+ "finish_reason": completion.choices[0].finish_reason,
831
+ "usage": completion.usage,
832
+ }
833
+ )
834
833
- # test seeded random sampling with multiple prompts
834
- completion = client.completions.create(
835
- model=model, prompt=[prompt, prompt], max_tokens=5, seed=33, temperature=1.0
836
- )
835
+ # test seeded random sampling with multiple prompts
836
+ completion = client.completions.create(
837
+ model=model,
838
+ prompt=[prompt, prompt],
839
+ max_tokens=5,
840
+ seed=33,
841
+ temperature=1.0,
842
+ )
843
838
- results.append(
839
- {
840
- "test": "seeded_sampling",
841
- "text": [choice.text for choice in completion.choices],
842
- "finish_reason": [choice.finish_reason for choice in completion.choices],
843
- "usage": completion.usage,
844
- }
845
- )
844
+ results.append(
845
+ {
846
+ "test": "seeded_sampling",
847
+ "text": [choice.text for choice in completion.choices],
848
+ "finish_reason": [
849
+ choice.finish_reason for choice in completion.choices
850
+ ],
851
+ "usage": completion.usage,
852
+ }
853
+ )
854
855
# test simple list
856
batch = client.completions.create(
@@ -1037,6 +1045,7 @@ def compare_two_settings(
1045
*,
1046
method: str = "generate",
1047
max_wait_seconds: float | None = None,
1048
+ include_seeded_sampling: bool = True,
1049
) -> None:
1050
"""
1051
Launch API server with two different sets of arguments/environments
@@ -1048,6 +1057,8 @@ def compare_two_settings(
1057
arg2: The second set of arguments to pass to the API server.
1058
env1: The first set of environment variables to pass to the API server.
1059
env2: The second set of environment variables to pass to the API server.
1060
+ include_seeded_sampling: Whether to include temperature=1.0 seeded
1061
+ sampling checks in the default generate comparison.
1062
"""
1063
1064
compare_all_settings(
@@ -1056,6 +1067,7 @@ def compare_two_settings(
1067
[env1, env2],
1068
method=method,
1069
max_wait_seconds=max_wait_seconds,
1070
+ include_seeded_sampling=include_seeded_sampling,
1071
)
1072
1073
@@ -1066,6 +1078,7 @@ def compare_all_settings(
1078
*,
1079
method: str = "generate",
1080
max_wait_seconds: float | None = None,
1081
+ include_seeded_sampling: bool = True,
1082
) -> None:
1083
"""
1084
Launch API server with several different sets of arguments/environments
@@ -1074,6 +1087,8 @@ def compare_all_settings(
1087
model: The model to test.
1088
all_args: A list of argument lists to pass to the API server.
1089
all_envs: A list of environment dictionaries to pass to the API server.
1090
+ include_seeded_sampling: Whether to include temperature=1.0 seeded
1091
+ sampling checks in the default generate comparison.
1092
"""
1093
1094
trust_remote_code = False
@@ -1134,7 +1149,13 @@ def compare_all_settings(
1149
)
1150
1151
if method == "generate":
1137
- results += _test_completion(client, model, prompt, token_ids)
1152
+ results += _test_completion(
1153
+ client,
1154
+ model,
1155
+ prompt,
1156
+ token_ids,
1157
+ include_seeded_sampling=include_seeded_sampling,
1158
+ )
1159
elif method == "generate_close":
1160
results += _test_completion_close(client, model, prompt)
1161
elif method == "generate_chat":