fb1a8f932a · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -198,8 +198,9 @@ class BenchmarkDataset(ABC):

198

199 @abstractmethod

200 def sample(self, tokenizer: PreTrainedTokenizerBase,

201 - num_requests: int,

202 - request_id_prefix: str = "") -> list[SampleRequest]:

201 + num_requests: int,

202 + request_id_prefix: str = "",

203 + no_oversample: bool = False) -> list[SampleRequest]:

204 """

205 Abstract method to generate sample requests from the dataset.

206

@@ -224,6 +225,7 @@ class BenchmarkDataset(ABC):

225 requests: list[SampleRequest],

226 num_requests: int,

227 request_id_prefix: str = "",

228 + no_oversample: bool = False,

229 ) -> None:

230 """

231 Oversamples the list of requests if its size is less than the desired

@@ -236,6 +238,11 @@ class BenchmarkDataset(ABC):

238 request_id_prefix (str) The prefix of the request ids.

239

240 """

241 + if no_oversample:

242 + logger.info("Skipping oversampling. " \

243 + "Total samples: %d.", len(requests))

244 + return

245 +

246 if len(requests) < num_requests:

247 random.seed(self.random_seed)

248 additional = deepcopy(

@@ -405,6 +412,7 @@ class RandomDataset(BenchmarkDataset):

412 tokenizer: PreTrainedTokenizerBase,

413 num_requests: int,

414 request_id_prefix: str = "",

415 + no_oversample: bool = False,

416 prefix_len: int = DEFAULT_PREFIX_LEN,

417 range_ratio: float = DEFAULT_RANGE_RATIO,

418 input_len: int = DEFAULT_INPUT_LEN,

@@ -832,6 +840,7 @@ class RandomMultiModalDataset(RandomDataset):

840 tokenizer: PreTrainedTokenizerBase,

841 num_requests: int,

842 request_id_prefix: str = "",

843 + no_oversample: bool = False,

844 prefix_len: int = RandomDataset.DEFAULT_PREFIX_LEN,

845 range_ratio: float = RandomDataset.DEFAULT_RANGE_RATIO,

846 input_len: int = RandomDataset.DEFAULT_INPUT_LEN,

@@ -959,6 +968,7 @@ class ShareGPTDataset(BenchmarkDataset):

968 output_len: Optional[int] = None,

969 enable_multimodal_chat: bool = False,

970 request_id_prefix: str = "",

971 + no_oversample: bool = False,

972 **kwargs,

973 ) -> list:

974 samples: list = []

@@ -1002,7 +1012,10 @@ class ShareGPTDataset(BenchmarkDataset):

1012 request_id=request_id_prefix + str(ind),

1013 ))

1014 ind += 1

1005 - self.maybe_oversample_requests(samples, num_requests, request_id_prefix)

1015 + self.maybe_oversample_requests(samples,

1016 + num_requests,

1017 + request_id_prefix,

1018 + no_oversample)

1019 return samples

1020

1021

@@ -1036,6 +1049,12 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1049 help="Path to the sharegpt/sonnet dataset. "

1050 "Or the huggingface dataset ID if using HF dataset.",

1051 )

1052 + parser.add_argument(

1053 + "--no-oversample",

1054 + action="store_true",

1055 + help="Do not oversample if the dataset has " \

1056 + "fewer samples than num-prompts.",

1057 + )

1058

1059 # group for dataset specific arguments

1060 custom_group = parser.add_argument_group("custom dataset options")

@@ -1322,6 +1341,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1341 output_len=args.custom_output_len,

1342 skip_chat_template=args.custom_skip_chat_template,

1343 request_id_prefix=args.request_id_prefix,

1344 + no_oversample=args.no_oversample,

1345 )

1346

1347 elif args.dataset_name == "sonnet":

@@ -1336,6 +1356,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1356 tokenizer=tokenizer,

1357 return_prompt_formatted=False,

1358 request_id_prefix=args.request_id_prefix,

1359 + no_oversample=args.no_oversample,

1360 )

1361 else:

1362 assert tokenizer.chat_template or tokenizer.default_chat_template, (

@@ -1348,6 +1369,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1369 tokenizer=tokenizer,

1370 return_prompt_formatted=True,

1371 request_id_prefix=args.request_id_prefix,

1372 + no_oversample=args.no_oversample,

1373 )

1374

1375 elif args.dataset_name == "hf":

@@ -1443,6 +1465,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1465 tokenizer=tokenizer,

1466 output_len=args.hf_output_len,

1467 request_id_prefix=args.request_id_prefix,

1468 + no_oversample=args.no_oversample,

1469 **hf_kwargs

1470 )

1471

@@ -1456,6 +1479,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1479 tokenizer=tokenizer,

1480 output_len=args.spec_bench_output_len,

1481 request_id_prefix=args.request_id_prefix,

1482 + no_oversample=args.no_oversample,

1483 ),

1484 "sharegpt": lambda: ShareGPTDataset(

1485 random_seed=args.seed, dataset_path=args.dataset_path

@@ -1464,6 +1488,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1488 num_requests=args.num_prompts,

1489 output_len=args.sharegpt_output_len,

1490 request_id_prefix=args.request_id_prefix,

1491 + no_oversample=args.no_oversample,

1492 ),

1493 "burstgpt": lambda: BurstGPTDataset(

1494 random_seed=args.seed, dataset_path=args.dataset_path

@@ -1471,6 +1496,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1496 tokenizer=tokenizer,

1497 num_requests=args.num_prompts,

1498 request_id_prefix=args.request_id_prefix,

1499 + no_oversample=args.no_oversample,

1500 ),

1501 "random": lambda: RandomDataset(

1502 random_seed=args.seed, dataset_path=args.dataset_path

@@ -1483,6 +1509,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1509 range_ratio=args.random_range_ratio,

1510 request_id_prefix=args.request_id_prefix,

1511 batchsize=args.random_batch_size,

1512 + no_oversample=args.no_oversample,

1513 ),

1514 "random-mm":

1515 lambda: RandomMultiModalDataset(

@@ -1499,6 +1526,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1526 num_mm_items_range_ratio=args.random_mm_num_mm_items_range_ratio,

1527 bucket_config=args.random_mm_bucket_config,

1528 request_id_prefix=args.request_id_prefix,

1529 + no_oversample=args.no_oversample,

1530 ),

1531 "prefix_repetition":

1532 lambda: PrefixRepetitionRandomDataset(

@@ -1511,6 +1539,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1539 num_prefixes=args.prefix_repetition_num_prefixes,

1540 output_len=args.prefix_repetition_output_len,

1541 request_id_prefix=args.request_id_prefix,

1542 + no_oversample=args.no_oversample,

1543 ),

1544 }

1545

@@ -1592,6 +1621,7 @@ class CustomDataset(BenchmarkDataset):

1621 enable_multimodal_chat: bool = False,

1622 skip_chat_template: bool = False,

1623 request_id_prefix: str = "",

1624 + no_oversample: bool = False,

1625 **kwargs,

1626 ) -> list:

1627 # load all data if needed

@@ -1628,7 +1658,7 @@ class CustomDataset(BenchmarkDataset):

1658 request_id=request_id_prefix + str(i),

1659 ))

1660 self.maybe_oversample_requests(sampled_requests, num_requests,

1631 - request_id_prefix)

1661 + request_id_prefix, no_oversample)

1662

1663 return sampled_requests

1664

@@ -1719,6 +1749,7 @@ class SonnetDataset(BenchmarkDataset):

1749 output_len: int = DEFAULT_OUTPUT_LEN,

1750 return_prompt_formatted: bool = False,

1751 request_id_prefix: str = "",

1752 + no_oversample: bool = False,

1753 **kwargs,

1754 ) -> list:

1755 # Calculate average token length for a poem line.

@@ -1814,6 +1845,7 @@ class BurstGPTDataset(BenchmarkDataset):

1845 max_loras: Optional[int] = None,

1846 lora_path: Optional[str] = None,

1847 request_id_prefix: str = "",

1848 + no_oversample: bool = False,

1849 **kwargs,

1850 ) -> list[SampleRequest]:

1851 samples = []

@@ -1893,6 +1925,7 @@ class ConversationDataset(HuggingFaceDataset):

1925 output_len: Optional[int] = None,

1926 enable_multimodal_chat: bool = False,

1927 request_id_prefix: str = "",

1928 + no_oversample: bool = False,

1929 **kwargs) -> list:

1930 # Filter examples with at least 2 conversations

1931 filtered_data = self.data.filter(

@@ -1934,7 +1967,7 @@ class ConversationDataset(HuggingFaceDataset):

1967 ))

1968 ind += 1

1969 self.maybe_oversample_requests(sampled_requests, num_requests,

1937 - request_id_prefix)

1970 + request_id_prefix, no_oversample)

1971 return sampled_requests

1972

1973

@@ -1964,6 +1997,7 @@ class VisionArenaDataset(HuggingFaceDataset):

1997 output_len: Optional[int] = None,

1998 enable_multimodal_chat: bool = False,

1999 request_id_prefix: str = "",

2000 + no_oversample: bool = False,

2001 **kwargs,

2002 ) -> list:

2003 output_len = (output_len

@@ -1993,7 +2027,7 @@ class VisionArenaDataset(HuggingFaceDataset):

2027 request_id=request_id_prefix + str(i),

2028 ))

2029 self.maybe_oversample_requests(sampled_requests, num_requests,

1996 - request_id_prefix)

2030 + request_id_prefix, no_oversample)

2031 return sampled_requests

2032

2033

@@ -2023,6 +2057,7 @@ class InstructCoderDataset(HuggingFaceDataset):

2057 output_len: Optional[int] = None,

2058 enable_multimodal_chat: bool = False,

2059 request_id_prefix: str = "",

2060 + no_oversample: bool = False,

2061 **kwargs) -> list:

2062 output_len = (output_len

2063 if output_len is not None else self.DEFAULT_OUTPUT_LEN)

@@ -2054,7 +2089,7 @@ class InstructCoderDataset(HuggingFaceDataset):

2089 request_id=request_id_prefix + str(i),

2090 ))

2091 self.maybe_oversample_requests(sampled_requests, num_requests,

2057 - request_id_prefix)

2092 + request_id_prefix, no_oversample)

2093 return sampled_requests

2094

2095

@@ -2085,6 +2120,7 @@ class MTBenchDataset(HuggingFaceDataset):

2120 output_len: Optional[int] = None,

2121 enable_multimodal_chat: bool = False,

2122 request_id_prefix: str = "",

2123 + no_oversample: bool = False,

2124 **kwargs,

2125 ) -> list:

2126 output_len = (output_len

@@ -2115,7 +2151,7 @@ class MTBenchDataset(HuggingFaceDataset):

2151 request_id=request_id_prefix + str(i),

2152 ))

2153 self.maybe_oversample_requests(sampled_requests, num_requests,

2118 - request_id_prefix)

2154 + request_id_prefix, no_oversample)

2155 return sampled_requests

2156

2157

@@ -2149,6 +2185,7 @@ class BlazeditDataset(HuggingFaceDataset):

2185 num_requests: int,

2186 output_len: Optional[int] = None,

2187 request_id_prefix: str = "",

2188 + no_oversample: bool = False,

2189 min_distance: float = 0.0,

2190 max_distance: float = 1.0,

2191 **kwargs,

@@ -2202,7 +2239,7 @@ Please generate the new code file in the "New file" section below.""" # noqa: E5

2239 request_id=request_id_prefix + str(i),

2240 ))

2241 self.maybe_oversample_requests(sampled_requests, num_requests,

2205 - request_id_prefix)

2242 + request_id_prefix, no_oversample)

2243

2244 return sampled_requests

2245

@@ -2226,6 +2263,7 @@ class AIMODataset(HuggingFaceDataset):

2263 num_requests: int,

2264 output_len: Optional[int] = None,

2265 request_id_prefix: str = "",

2266 + no_oversample: bool = False,

2267 **kwargs) -> list:

2268 sampled_requests = []

2269 ind = 0

@@ -2258,7 +2296,7 @@ class AIMODataset(HuggingFaceDataset):

2296 ))

2297 ind += 1

2298 self.maybe_oversample_requests(sampled_requests, num_requests,

2261 - request_id_prefix)

2299 + request_id_prefix, no_oversample)

2300 return sampled_requests

2301

2302

@@ -2330,6 +2368,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):

2368

2369 def sample(self, tokenizer: PreTrainedTokenizerBase, num_requests: int,

2370 request_id_prefix: str = "",

2371 + no_oversample: bool = False,

2372 **kwargs):

2373 formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)

2374 if formatting_prompt_func is None:

@@ -2347,7 +2386,10 @@ class NextEditPredictionDataset(HuggingFaceDataset):

2386 ))

2387 if len(samples) >= num_requests:

2388 break

2350 - self.maybe_oversample_requests(samples, num_requests, request_id_prefix)

2389 + self.maybe_oversample_requests(samples,

2390 + num_requests,

2391 + request_id_prefix,

2392 + no_oversample)

2393 return samples

2394

2395

@@ -2398,6 +2440,7 @@ class ASRDataset(HuggingFaceDataset):

2440 num_requests: int,

2441 output_len: Optional[int] = None,

2442 request_id_prefix: str = "",

2443 + no_oversample: bool = False,

2444 **kwargs,

2445 ) -> list:

2446 output_len = (output_len

@@ -2436,7 +2479,7 @@ class ASRDataset(HuggingFaceDataset):

2479 skipped,

2480 )

2481 self.maybe_oversample_requests(sampled_requests, num_requests,

2439 - request_id_prefix)

2482 + request_id_prefix, no_oversample)

2483 return sampled_requests

2484

2485

@@ -2474,6 +2517,7 @@ class MLPerfDataset(HuggingFaceDataset):

2517 num_requests: int,

2518 output_len: Optional[int] = None,

2519 request_id_prefix: str = "",

2520 + no_oversample: bool = False,

2521 **kwargs,

2522 ) -> list[SampleRequest]:

2523 # Force dynamic output length based on reference completion.

@@ -2520,7 +2564,7 @@ class MLPerfDataset(HuggingFaceDataset):

2564 ind += 1

2565

2566 self.maybe_oversample_requests(sampled_requests, num_requests,

2523 - request_id_prefix)

2567 + request_id_prefix, no_oversample)

2568 return sampled_requests

2569

2570

@@ -2554,6 +2598,7 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):

2598 num_prefixes: int = DEFAULT_NUM_PREFIXES,

2599 output_len: int = DEFAULT_OUTPUT_LEN,

2600 request_id_prefix: str = "",

2601 + no_oversample: bool = False,

2602 **kwargs,

2603 ) -> list[SampleRequest]:

2604 vocab_size = tokenizer.vocab_size

[Benchmark] Add option to skip oversampling in benchmark (#24457)