[MISC][Bench] refactor throughput and reuse serve's get samples (#50981)
Signed-off-by: JaredforReal <w13431838023@gmail.com>
Jared Wen committed
Aug 6, 2026 at 18:31 UTC
e07532b0358784069ed37c36276a14b697cb3d47
3 files changed
+228
-231
tests/benchmarks/test_throughput_cli.py
+128
@@ -1,13 +1,25 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Tests for ``vllm bench throughput`` CLI.
4
+
5
+Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
6
+LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
7
+(serve has no analogue), keeping the two benchmarks aligned without duplicating
8
+serve-side dataset coverage.
9
+"""
10
+
11
import subprocess
12
+from types import SimpleNamespace
13
14
import pytest
15
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
16
17
from vllm.benchmarks.datasets import SampleRequest
18
from vllm.benchmarks.throughput import (
19
_run_vllm_chat_requests,
20
add_cli_args,
21
+ assign_loras,
22
+ get_requests,
23
)
24
from vllm.utils.argparse_utils import FlexibleArgumentParser
25
@@ -102,3 +114,119 @@ def test_vllm_chat_requests_include_multimodal_content():
114
}
115
]
116
]
117
+
118
+
119
+@pytest.fixture(scope="session")
120
+def hf_tokenizer() -> PreTrainedTokenizerBase:
121
+ # Small, commonly available tokenizer.
122
+ return AutoTokenizer.from_pretrained("openai-community/gpt2")
123
+
124
+
125
+# -----------------------------
126
+# LoRA assignment (throughput-only post-processing)
127
+# -----------------------------
128
+
129
+
130
+def _sr(i: int) -> SampleRequest:
131
+ return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
132
+
133
+
134
+def _lora_args(
135
+ *,
136
+ lora_path: str | None,
137
+ max_loras: int,
138
+ lora_assignment: str = "random",
139
+) -> SimpleNamespace:
140
+ return SimpleNamespace(
141
+ lora_path=lora_path,
142
+ max_loras=max_loras,
143
+ lora_assignment=lora_assignment,
144
+ )
145
+
146
+
147
+@pytest.mark.benchmark
148
+def test_assign_loras_noop_without_lora_path() -> None:
149
+ """No lora_path -> requests returned unchanged, no LoRA attached."""
150
+ reqs = [_sr(i) for i in range(5)]
151
+ out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
152
+ assert out is reqs
153
+ assert all(r.lora_request is None for r in out)
154
+
155
+
156
+@pytest.mark.benchmark
157
+def test_assign_loras_round_robin() -> None:
158
+ """Round-robin assigns deterministic index % max_loras + 1 IDs."""
159
+ reqs = [_sr(i) for i in range(6)]
160
+ out = assign_loras(
161
+ reqs,
162
+ _lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
163
+ )
164
+ assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
165
+
166
+
167
+@pytest.mark.benchmark
168
+def test_assign_loras_random_in_range() -> None:
169
+ """Random assignment stays in [1, max_loras] and covers all IDs."""
170
+ reqs = [_sr(i) for i in range(200)]
171
+ out = assign_loras(
172
+ reqs,
173
+ _lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
174
+ )
175
+ ids = [r.lora_request.lora_int_id for r in out]
176
+ assert all(1 <= i <= 5 for i in ids)
177
+ assert len(set(ids)) == 5
178
+
179
+
180
+# -----------------------------
181
+# Issue #50838: MMVU via throughput
182
+# -----------------------------
183
+
184
+
185
+@pytest.mark.benchmark
186
+def test_get_requests_resolves_mmvu(
187
+ monkeypatch: pytest.MonkeyPatch,
188
+ hf_tokenizer: PreTrainedTokenizerBase,
189
+) -> None:
190
+ """Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
191
+
192
+ Previously throughput's validate_args rejected MMVU ("is not supported by
193
+ hf dataset"). It now resolves through the shared get_samples. The HF
194
+ download is stubbed so the resolution + dispatch path is exercised without
195
+ network access.
196
+ """
197
+ import vllm.benchmarks.datasets.datasets as dsmod
198
+
199
+ class _StubMMVUDataset:
200
+ # MMVU is multimodal in content but, like the real class, does not set
201
+ # IS_MULTIMODAL -- so it is not subject to the backend gate.
202
+ IS_MULTIMODAL = False
203
+ SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
204
+
205
+ def __init__(self, **kwargs):
206
+ self.__dict__.update(kwargs)
207
+
208
+ def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
209
+ return [
210
+ SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
211
+ for _ in range(num_requests)
212
+ ]
213
+
214
+ monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
215
+
216
+ parser = FlexibleArgumentParser()
217
+ add_cli_args(parser)
218
+ args = parser.parse_args(
219
+ [
220
+ "--dataset-name",
221
+ "hf",
222
+ "--dataset-path",
223
+ "yale-nlp/MMVU",
224
+ "--backend",
225
+ "vllm-chat",
226
+ "--num-prompts",
227
+ "3",
228
+ ]
229
+ )
230
+ requests = get_requests(args, hf_tokenizer)
231
+ assert len(requests) == 3
232
+ assert all(isinstance(r, SampleRequest) for r in requests)
vllm/benchmarks/datasets/datasets.py
+28
-16
@@ -164,8 +164,8 @@ class BenchmarkDataset(ABC):
164
# TODO (jenniferzhao): add support for downloading data
165
raise NotImplementedError("load_data must be implemented in subclasses.")
166
167
+ @staticmethod
168
def get_random_lora_request(
168
- self,
169
max_loras: int | None = None,
170
lora_path: str | None = None,
171
) -> LoRARequest | None:
@@ -197,8 +197,8 @@ class BenchmarkDataset(ABC):
197
)
198
return lora_request
199
200
+ @staticmethod
201
def get_round_robin_lora_request(
201
- self,
202
index: int,
203
max_loras: int | None = None,
204
lora_path: str | None = None,
@@ -232,8 +232,8 @@ class BenchmarkDataset(ABC):
232
)
233
return lora_request
234
235
+ @staticmethod
236
def get_lora_request(
236
- self,
237
index: int,
238
max_loras: int | None = None,
239
lora_path: str | None = None,
@@ -254,10 +254,12 @@ class BenchmarkDataset(ABC):
254
(or `None` if not applicable).
255
"""
256
if lora_assignment == "round-robin":
257
- return self.get_round_robin_lora_request(
257
+ return BenchmarkDataset.get_round_robin_lora_request(
258
index=index, max_loras=max_loras, lora_path=lora_path
259
)
260
- return self.get_random_lora_request(max_loras=max_loras, lora_path=lora_path)
260
+ return BenchmarkDataset.get_random_lora_request(
261
+ max_loras=max_loras, lora_path=lora_path
262
+ )
263
264
@abstractmethod
265
def sample(
@@ -2091,7 +2093,12 @@ def _parse_range_ratio(value: str) -> RangeRatio:
2093
return json.loads(value)
2094
2095
2094
-def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
2096
+def get_samples(
2097
+ args,
2098
+ tokenizer: TokenizerLike | None,
2099
+ *,
2100
+ multimodal_backends: tuple[str, ...] = ("openai-chat", "openai-audio"),
2101
+) -> list[SampleRequest]:
2102
if not hasattr(args, "request_id_prefix"):
2103
args.request_id_prefix = ""
2104
@@ -2310,15 +2317,17 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
2317
"like to add support for additional dataset formats."
2318
)
2319
2313
- if dataset_class.IS_MULTIMODAL and not (
2314
- args.backend in ("openai-chat", "openai-audio")
2315
- or "embeddings-" in args.backend
2320
+ if (
2321
+ dataset_class.IS_MULTIMODAL
2322
+ and args.backend not in multimodal_backends
2323
+ and "embeddings-" not in args.backend
2324
):
2317
- # multi-modal benchmark is only available on OpenAI Chat
2318
- # endpoint-type.
2325
+ # multi-modal benchmark is only available on chat-style backends;
2326
+ # which ones are allowed is caller-controlled (serve uses the
2327
+ # OpenAI endpoints, throughput uses vllm-chat).
2328
raise ValueError(
2320
- "Multi-modal content is only supported on 'openai-chat' and "
2321
- "'openai-audio' backends."
2329
+ f"Multi-modal content is not supported on backend "
2330
+ f"{args.backend!r}; use one of {sorted(multimodal_backends)}."
2331
)
2332
assert tokenizer is not None, (
2333
"Tokenizer must be initialized for the 'hf' dataset."
@@ -2474,10 +2483,13 @@ def get_samples(args, tokenizer: TokenizerLike | None) -> list[SampleRequest]:
2483
2484
try:
2485
# Enforce endpoint compatibility for multimodal datasets.
2477
- if args.dataset_name == "random-mm" and args.backend not in ["openai-chat"]:
2486
+ if (
2487
+ args.dataset_name == "random-mm"
2488
+ and args.backend not in multimodal_backends
2489
+ ):
2490
raise ValueError(
2479
- "Multi-modal content (images) is only supported on "
2480
- "'openai-chat' backend."
2491
+ f"Multi-modal content (images) is not supported on backend "
2492
+ f"{args.backend!r}; use one of {sorted(multimodal_backends)}."
2493
)
2494
input_requests = dataset_mapping[args.dataset_name]()
2495
except KeyError as err:
vllm/benchmarks/throughput.py
+72
-215
@@ -16,24 +16,11 @@ from tqdm import tqdm
16
from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase
17
18
from vllm.benchmarks.datasets import (
19
- AIMODataset,
20
- ASRDataset,
19
BenchmarkDataset,
22
- BurstGPTDataset,
23
- ConversationDataset,
24
- CustomAudioDataset,
25
- InstructCoderDataset,
26
- MultiModalConversationDataset,
27
- PrefixRepetitionRandomDataset,
28
- RandomDataset,
29
- RandomDatasetForReranking,
30
- RandomMultiModalDataset,
20
SampleRequest,
32
- ShareGPTDataset,
33
- SonnetDataset,
34
- VisionArenaDataset,
21
add_random_dataset_base_args,
22
add_random_multimodal_dataset_args,
23
+ get_samples,
24
)
25
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
26
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
@@ -536,181 +523,56 @@ def save_to_pytorch_benchmark_format(
523
write_to_json(pt_file, pt_records)
524
525
526
+def _to_serve_args(args) -> argparse.Namespace:
527
+ """Translate throughput args into a namespace the shared get_samples reads.
528
+
529
+ ``get_samples`` (used by ``bench serve``) expects ~45 attributes; the
530
+ throughput CLI exposes most of them under the same names, and this adapter
531
+ fills the rest while preserving throughput's existing flag names so no
532
+ script breaks.
533
+
534
+ Args:
535
+ args: parsed throughput CLI namespace.
536
+
537
+ Returns:
538
+ A namespace satisfying get_samples's attribute reads.
539
+ """
540
+ d = vars(args).copy()
541
+ # random_*: prefer --random-* over legacy --input/output/prefix-len.
542
+ d["random_input_len"] = getattr(args, "random_input_len", None) or args.input_len
543
+ d["random_output_len"] = getattr(args, "random_output_len", None) or args.output_len
544
+ d["random_prefix_len"] = getattr(args, "random_prefix_len", None) or args.prefix_len
545
+ # --output-len maps to the per-dataset output-len entry points get_samples
546
+ # reads. None passes through (each dataset applies its own default),
547
+ # matching prior throughput behaviour of omitting output_len when unset.
548
+ d["hf_output_len"] = args.output_len
549
+ d["sharegpt_output_len"] = args.output_len
550
+ # sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults.
551
+ d["sonnet_input_len"] = args.input_len if args.input_len is not None else 550
552
+ d["sonnet_output_len"] = args.output_len if args.output_len is not None else 150
553
+ d["sonnet_prefix_len"] = args.prefix_len
554
+ # Explicit --enable-multimodal-chat wins; otherwise auto-enable for the
555
+ # multimodal chat backend (preserves today's vllm-chat handling).
556
+ d["enable_multimodal_chat"] = bool(
557
+ getattr(args, "enable_multimodal_chat", False) or args.backend == "vllm-chat"
558
+ )
559
+ # serve-only attrs throughput never exposed; keep serve's defaults.
560
+ d.setdefault("disable_shuffle", False)
561
+ d.setdefault("skip_chat_template", False)
562
+ d.setdefault("no_stream", False)
563
+ d.setdefault("request_id_prefix", "")
564
+ d.setdefault("chat_template_kwargs", None)
565
+ return argparse.Namespace(**d)
566
+
567
+
568
def get_requests(args, tokenizer):
540
- # Common parameters for all dataset types.
541
- dataset_cls: type[BenchmarkDataset]
542
- common_kwargs = {
543
- "dataset_path": args.dataset_path,
544
- "random_seed": args.seed,
545
- }
546
- sample_kwargs = {
547
- "tokenizer": tokenizer,
548
- "lora_path": args.lora_path,
549
- "max_loras": args.max_loras,
550
- "lora_assignment": getattr(args, "lora_assignment", "random"),
551
- "num_requests": args.num_prompts,
552
- "no_oversample": getattr(args, "no_oversample", False),
553
- }
554
-
555
- if args.dataset_name == "random" or (
556
- args.dataset_path is None
557
- and args.dataset_name not in {"prefix_repetition", "random-mm", "random-rerank"}
558
- ):
559
- sample_kwargs["range_ratio"] = args.random_range_ratio
560
- # prefer random_* arguments, fall back to regular arguments
561
- random_prefix_len = getattr(args, "random_prefix_len", None)
562
- sample_kwargs["prefix_len"] = (
563
- random_prefix_len if random_prefix_len is not None else args.prefix_len
564
- )
565
- random_input_len = getattr(args, "random_input_len", None)
566
- sample_kwargs["input_len"] = (
567
- random_input_len if random_input_len is not None else args.input_len
568
- )
569
- random_output_len = getattr(args, "random_output_len", None)
570
- sample_kwargs["output_len"] = (
571
- random_output_len if random_output_len is not None else args.output_len
572
- )
573
- dataset_cls = RandomDataset
574
- elif args.dataset_name == "sharegpt":
575
- dataset_cls = ShareGPTDataset
576
- if args.backend == "vllm-chat":
577
- sample_kwargs["enable_multimodal_chat"] = True
578
- if args.output_len is not None:
579
- sample_kwargs["output_len"] = args.output_len
580
- elif args.dataset_name == "sonnet":
581
- assert tokenizer.chat_template or tokenizer.default_chat_template, (
582
- "Tokenizer/model must have chat template for sonnet dataset."
583
- )
584
- dataset_cls = SonnetDataset
585
- sample_kwargs["prefix_len"] = args.prefix_len
586
- sample_kwargs["return_prompt_formatted"] = True
587
- if args.input_len is not None:
588
- sample_kwargs["input_len"] = args.input_len
589
- if args.output_len is not None:
590
- sample_kwargs["output_len"] = args.output_len
591
- elif args.dataset_name == "burstgpt":
592
- dataset_cls = BurstGPTDataset
593
- elif args.dataset_name == "custom_audio":
594
- dataset_cls = CustomAudioDataset
595
- sample_kwargs["enable_multimodal_chat"] = getattr(
596
- args, "enable_multimodal_chat", False
597
- )
598
- custom_output_len = getattr(args, "custom_output_len", None)
599
- if custom_output_len is not None:
600
- sample_kwargs["output_len"] = custom_output_len
601
- elif args.output_len is not None:
602
- sample_kwargs["output_len"] = args.output_len
603
- elif args.dataset_name == "hf":
604
- if args.output_len is not None:
605
- sample_kwargs["output_len"] = args.output_len
606
- common_kwargs["hf_name"] = args.hf_name
607
- if (
608
- args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS
609
- or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS
610
- ):
611
- dataset_cls = VisionArenaDataset
612
- common_kwargs["dataset_subset"] = None
613
- common_kwargs["dataset_split"] = "train"
614
- sample_kwargs["enable_multimodal_chat"] = True
615
- elif (
616
- args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS
617
- or args.hf_name in InstructCoderDataset.SUPPORTED_DATASET_PATHS
618
- ):
619
- dataset_cls = InstructCoderDataset
620
- common_kwargs["dataset_split"] = "train"
621
- elif (
622
- args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
623
- or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
624
- ):
625
- dataset_cls = MultiModalConversationDataset
626
- common_kwargs["dataset_subset"] = args.hf_subset
627
- common_kwargs["dataset_split"] = args.hf_split
628
- sample_kwargs["enable_multimodal_chat"] = True
629
- elif (
630
- args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS
631
- or args.hf_name in ConversationDataset.SUPPORTED_DATASET_PATHS
632
- ):
633
- dataset_cls = ConversationDataset
634
- common_kwargs["dataset_subset"] = args.hf_subset
635
- common_kwargs["dataset_split"] = args.hf_split
636
- sample_kwargs["enable_multimodal_chat"] = True
637
- elif (
638
- args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS
639
- or args.hf_name in AIMODataset.SUPPORTED_DATASET_PATHS
640
- ):
641
- dataset_cls = AIMODataset
642
- common_kwargs["dataset_subset"] = None
643
- common_kwargs["dataset_split"] = "train"
644
- elif (
645
- args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS
646
- or args.hf_name in ASRDataset.SUPPORTED_DATASET_PATHS
647
- ):
648
- dataset_cls = ASRDataset
649
- common_kwargs["dataset_subset"] = args.hf_subset
650
- common_kwargs["dataset_split"] = args.hf_split
651
- sample_kwargs["asr_min_audio_len_sec"] = args.asr_min_audio_len_sec
652
- sample_kwargs["asr_max_audio_len_sec"] = args.asr_max_audio_len_sec
653
- elif args.dataset_name == "prefix_repetition":
654
- dataset_cls = PrefixRepetitionRandomDataset
655
- sample_kwargs["prefix_len"] = args.prefix_repetition_prefix_len
656
- sample_kwargs["suffix_len"] = args.prefix_repetition_suffix_len
657
- sample_kwargs["num_prefixes"] = args.prefix_repetition_num_prefixes
658
- sample_kwargs["output_len"] = args.prefix_repetition_output_len
659
- elif args.dataset_name == "random-mm":
660
- dataset_cls = RandomMultiModalDataset
661
- # prefer random_* arguments, fall back to regular arguments
662
- random_input_len = getattr(args, "random_input_len", None)
663
- sample_kwargs["input_len"] = (
664
- random_input_len
665
- if random_input_len is not None
666
- else getattr(args, "input_len", None)
667
- )
668
- random_output_len = getattr(args, "random_output_len", None)
669
- sample_kwargs["output_len"] = (
670
- random_output_len
671
- if random_output_len is not None
672
- else getattr(args, "output_len", None)
673
- )
674
- sample_kwargs["base_items_per_request"] = getattr(
675
- args, "random_mm_base_items_per_request", None
676
- )
677
- sample_kwargs["num_mm_items_range_ratio"] = getattr(
678
- args, "random_mm_num_mm_items_range_ratio", None
679
- )
680
- sample_kwargs["limit_mm_per_prompt"] = getattr(
681
- args, "random_mm_limit_mm_per_prompt", None
682
- )
683
- sample_kwargs["bucket_config"] = getattr(args, "random_mm_bucket_config", None)
684
- sample_kwargs["enable_multimodal_chat"] = True
685
- random_prefix_len = getattr(args, "random_prefix_len", None)
686
- prefix_len = getattr(args, "prefix_len", None)
687
- sample_kwargs["prefix_len"] = (
688
- random_prefix_len if random_prefix_len is not None else prefix_len
689
- )
690
- sample_kwargs["range_ratio"] = args.random_range_ratio
691
- elif args.dataset_name == "random-rerank":
692
- dataset_cls = RandomDatasetForReranking
693
- # prefer random_* arguments, fall back to regular arguments
694
- random_input_len = getattr(args, "random_input_len", None)
695
- sample_kwargs["input_len"] = (
696
- random_input_len
697
- if random_input_len is not None
698
- else getattr(args, "input_len", None)
699
- )
700
- random_output_len = getattr(args, "random_output_len", None)
701
- sample_kwargs["output_len"] = (
702
- random_output_len
703
- if random_output_len is not None
704
- else getattr(args, "output_len", None)
705
- )
706
- sample_kwargs["batchsize"] = getattr(args, "random_batch_size", 1)
707
- sample_kwargs["is_reranker"] = not getattr(args, "no_reranker", False)
708
- sample_kwargs["range_ratio"] = args.random_range_ratio
709
- else:
710
- raise ValueError(f"Unknown dataset name: {args.dataset_name}")
711
- # Remove None values
712
- sample_kwargs = {k: v for k, v in sample_kwargs.items() if v is not None}
713
- requests = dataset_cls(**common_kwargs).sample(**sample_kwargs)
569
+ serve_args = _to_serve_args(args)
570
+ # Throughput reuses bench serve's dataset dispatch. vllm-chat is the only
571
+ # multimodal-capable throughput backend, so it is the only one allowed past
572
+ # the multimodal gates inside get_samples; other backends raise there.
573
+ mm_backends = ("vllm-chat",) if args.backend == "vllm-chat" else ()
574
+ requests = get_samples(serve_args, tokenizer, multimodal_backends=mm_backends)
575
+ requests = assign_loras(requests, args)
576
requests = filter_requests_for_dp(requests, args.data_parallel_size)
577
return requests
578
@@ -732,6 +594,28 @@ def filter_requests_for_dp(requests, data_parallel_size):
594
]
595
596
597
+def assign_loras(requests, args):
598
+ """Attach a LoRA request to each sample (throughput-only post-processing).
599
+
600
+ The shared ``datasets.get_samples`` path carries no LoRA information, so
601
+ LoRA assignment is applied here, uniformly across every dataset type. No-op
602
+ when ``--lora-path`` is unset.
603
+ """
604
+ lora_path = getattr(args, "lora_path", None)
605
+ if not lora_path:
606
+ return requests
607
+ max_loras = args.max_loras
608
+ lora_assignment = getattr(args, "lora_assignment", "random")
609
+ for i, req in enumerate(requests):
610
+ req.lora_request = BenchmarkDataset.get_lora_request(
611
+ index=i,
612
+ max_loras=max_loras,
613
+ lora_path=lora_path,
614
+ lora_assignment=lora_assignment,
615
+ )
616
+ return requests
617
+
618
+
619
def validate_args(args):
620
"""
621
Validate command-line arguments.
@@ -785,33 +669,6 @@ def validate_args(args):
669
since --dataset-name is not 'hf'.",
670
stacklevel=2,
671
)
788
- elif args.dataset_name == "hf":
789
- if args.dataset_path in (
790
- VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
791
- | MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
792
- | ConversationDataset.SUPPORTED_DATASET_PATHS
793
- ) or args.hf_name in (
794
- VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
795
- | MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
796
- | ConversationDataset.SUPPORTED_DATASET_PATHS
797
- ):
798
- assert args.backend == "vllm-chat", (
799
- f"{args.dataset_path} needs to use vllm-chat as the backend."
800
- )
801
- elif args.dataset_path in (
802
- InstructCoderDataset.SUPPORTED_DATASET_PATHS
803
- | AIMODataset.SUPPORTED_DATASET_PATHS
804
- | ASRDataset.SUPPORTED_DATASET_PATHS
805
- ) or args.hf_name in (
806
- InstructCoderDataset.SUPPORTED_DATASET_PATHS
807
- | AIMODataset.SUPPORTED_DATASET_PATHS
808
- | ASRDataset.SUPPORTED_DATASET_PATHS
809
- ):
810
- assert args.backend == "vllm", (
811
- f"{args.dataset_path} needs to use vllm as the backend."
812
- )
813
- else:
814
- raise ValueError(f"{args.dataset_path} is not supported by hf dataset.")
672
673
# --random-range-ratio: only used when dataset_name is 'random',
674
# 'random-mm', or 'random-rerank'