| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Download dataset from: |
| 4 | https://huggingface.co/datasets/philschmid/sharegpt-raw/blob/main/sharegpt_20230401_clean_lang_split.json |
| 5 | |
| 6 | Convert to OpenAI API: |
| 7 | export INPUT_FILE=sharegpt_20230401_clean_lang_split.json |
| 8 | python convert_sharegpt_to_openai.py $INPUT_FILE sharegpt_conv_128.json --max-items=128 |
| 9 | """ |
| 10 | |
| 11 | import argparse |
| 12 | import json |
| 13 | import random |
| 14 | from statistics import mean |
| 15 | from typing import Any |
| 16 | |
| 17 | import pandas as pd # type: ignore |
| 18 | import tqdm # type: ignore |
| 19 | from transformers import AutoTokenizer # type: ignore |
| 20 | |
| 21 | |
| 22 | def has_non_english_chars(text: str) -> bool: |
| 23 | return not text.isascii() |
| 24 | |
| 25 | |
| 26 | def content_is_valid( |
| 27 | content: str, min_content_len: int | None, max_content_len: int | None |
| 28 | ) -> bool: |
| 29 | if min_content_len and len(content) < min_content_len: |
| 30 | return False |
| 31 | |
| 32 | if max_content_len and len(content) > max_content_len: |
| 33 | return False |
| 34 | |
| 35 | return has_non_english_chars(content) |
| 36 | |
| 37 | |
| 38 | def print_stats( |
| 39 | conversations: "list[dict[Any, Any]]", tokenizer: AutoTokenizer | None = None |
| 40 | ) -> None: |
| 41 | # Collect statistics |
| 42 | stats = [] |
| 43 | |
| 44 | print("\nCollecting statistics...") |
| 45 | for item in tqdm.tqdm(conversations): |
| 46 | # item has "id" and "messages" |
| 47 | messages = item["messages"] |
| 48 | |
| 49 | user_turns = 0 |
| 50 | assistant_turns = 0 |
| 51 | user_words = 0 |
| 52 | assistant_words = 0 |
| 53 | conv_chars = 0 |
| 54 | |
| 55 | user_tokens: list[int] = [] |
| 56 | assistant_tokens: list[int] = [] |
| 57 | |
| 58 | for m in messages: |
| 59 | content = m["content"] |
| 60 | conv_chars += len(content) |
| 61 | content_num_words = content.count(" ") + 1 |
| 62 | |
| 63 | num_tokens = 0 |
| 64 | if tokenizer: |
| 65 | num_tokens = len(tokenizer(m["content"]).input_ids) |
| 66 | |
| 67 | if m["role"] == "user": |
| 68 | user_turns += 1 |
| 69 | user_words += content_num_words |
| 70 | if tokenizer: |
| 71 | user_tokens.append(num_tokens) |
| 72 | |
| 73 | elif m["role"] == "assistant": |
| 74 | assistant_turns += 1 |
| 75 | assistant_words += content_num_words |
| 76 | if tokenizer: |
| 77 | assistant_tokens.append(num_tokens) |
| 78 | |
| 79 | # assert user_turns == assistant_turns, \ |
| 80 | # f"Invalid conversation ID {item['id']}" |
| 81 | |
| 82 | conv_words = user_words + assistant_words |
| 83 | item_stats = { |
| 84 | "user_turns": user_turns, |
| 85 | "assistant_turns": assistant_turns, |
| 86 | "user_words": user_words, |
| 87 | "assistant_words": assistant_words, |
| 88 | "conv_turns": len(messages), |
| 89 | "conv_words": conv_words, |
| 90 | "conv_characters": conv_chars, |
| 91 | } |
| 92 | |
| 93 | if len(user_tokens) > 0: |
| 94 | item_stats["user_tokens"] = int(mean(user_tokens)) |
| 95 | |
| 96 | if len(assistant_tokens) > 0: |
| 97 | item_stats["assistant_tokens"] = int(mean(assistant_tokens)) |
| 98 | |
| 99 | stats.append(item_stats) |
| 100 | |
| 101 | print("\nStatistics:") |
| 102 | percentiles = [0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 0.9999] |
| 103 | df = pd.DataFrame(stats) |
| 104 | print(df.describe(percentiles=percentiles).transpose()) |
| 105 | |
| 106 | |
| 107 | def convert_sharegpt_to_openai( |
| 108 | seed: int, |
| 109 | input_file: str, |
| 110 | output_file: str, |
| 111 | max_items: int | None, |
| 112 | min_content_len: int | None = None, |
| 113 | max_content_len: int | None = None, |
| 114 | min_turns: int | None = None, |
| 115 | max_turns: int | None = None, |
| 116 | model: str | None = None, |
| 117 | ) -> None: |
| 118 | if min_turns and max_turns: |
| 119 | assert min_turns <= max_turns |
| 120 | |
| 121 | if min_content_len and max_content_len: |
| 122 | # Verify that min is not larger than max if both were given |
| 123 | assert min_content_len <= max_content_len |
| 124 | |
| 125 | print( |
| 126 | f"Input parameters:\n{seed=}, {max_items=}, {min_content_len=}," |
| 127 | f" {max_content_len=}, {min_turns=}, {max_turns=}\n" |
| 128 | ) |
| 129 | |
| 130 | random.seed(seed) |
| 131 | |
| 132 | tokenizer = None |
| 133 | if model is not None: |
| 134 | print(f"Loading tokenizer from: {model}") |
| 135 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 136 | |
| 137 | # Read the ShareGPT JSON file |
| 138 | print(f"Reading file: {input_file}") |
| 139 | with open(input_file, encoding="utf-8") as f: |
| 140 | # Should be a list of dicts |
| 141 | # Each dict should have "id" (string) and "conversations" (list of dicts) |
| 142 | sharegpt_data = json.load(f) |
| 143 | |
| 144 | assert isinstance(sharegpt_data, list), "Input file should contain a list of dicts" |
| 145 | |
| 146 | print(f"Total items in input file: {len(sharegpt_data):,}") |
| 147 | |
| 148 | print(f"Shuffling dataset with seed {seed}") |
| 149 | random.shuffle(sharegpt_data) |
| 150 | |
| 151 | # Map conversation ID to the all the messages |
| 152 | conversation_parts: dict[str, list[Any]] = {} |
| 153 | |
| 154 | for item in tqdm.tqdm(sharegpt_data): |
| 155 | assert "id" in item, "Missing key 'id'" |
| 156 | assert "conversations" in item, "Missing key 'conversations'" |
| 157 | |
| 158 | # Conversation ID (e.g: "hiWPlMD") and part/session (0, 1, 2, etc.) |
| 159 | conv_id, _ = item["id"].split("_") |
| 160 | new_turns = item["conversations"] |
| 161 | |
| 162 | if conv_id not in conversation_parts: |
| 163 | # Start new conversation |
| 164 | conversation_parts[conv_id] = [] |
| 165 | elif len(conversation_parts[conv_id]) > 0 and len(new_turns) > 0: |
| 166 | prev_turns = conversation_parts[conv_id][-1] |
| 167 | if prev_turns[-1]["from"] == new_turns[0]["from"]: |
| 168 | new_turns = new_turns[1:] |
| 169 | |
| 170 | if len(new_turns) > 0: |
| 171 | # We assume that parts are in order in the ShareGPT dataset |
| 172 | conversation_parts[conv_id].append(new_turns) |
| 173 | |
| 174 | dataset: list[dict[str, Any]] = [] |
| 175 | for conv_id, conv_parts in conversation_parts.items(): |
| 176 | new_item = {"id": conv_id} |
| 177 | |
| 178 | conversations: list[dict[str, str]] = [] |
| 179 | |
| 180 | # Merge all parts |
| 181 | for conv_part in conv_parts: |
| 182 | conversations.extend(conv_part) |
| 183 | |
| 184 | if len(conversations) > 0: |
| 185 | new_item["conversations"] = conversations |
| 186 | dataset.append(new_item) |
| 187 | |
| 188 | print(f"Total unique conversations (IDs) in input file: {len(dataset):,}") |
| 189 | |
| 190 | # Final output data |
| 191 | final_openai_dataset: list[dict] = [] |
| 192 | |
| 193 | # Filter conversations from the ShareGPT dataset and convert to OpenAI format |
| 194 | for item in tqdm.tqdm(dataset): |
| 195 | messages: list[dict] = [] |
| 196 | |
| 197 | assert "id" in item, "Missing key 'id'" |
| 198 | assert "conversations" in item, "Missing key 'conversations'" |
| 199 | |
| 200 | conv_id = item["id"] |
| 201 | conversations = item["conversations"] |
| 202 | |
| 203 | if min_turns is not None and len(conversations) < min_turns: |
| 204 | # Skip short conversations |
| 205 | continue |
| 206 | |
| 207 | # Convert each message in the conversation, up to max_turns if specified |
| 208 | for i, turn in enumerate(conversations): |
| 209 | assert "from" in turn and "value" in turn, ( |
| 210 | f"Invalid conversation ID {conv_id} - missing 'from' or 'value'" |
| 211 | ) |
| 212 | |
| 213 | role = None |
| 214 | turn_from = turn["from"] |
| 215 | |
| 216 | if turn_from in {"human", "user"}: |
| 217 | role = "user" |
| 218 | elif turn_from in {"gpt", "bing", "chatgpt", "bard"}: |
| 219 | role = "assistant" |
| 220 | elif turn_from == "system": |
| 221 | role = "system" |
| 222 | |
| 223 | assert role is not None, ( |
| 224 | f"Invalid conversation ID {conv_id} - 'from'='{turn_from}' is invalid" |
| 225 | ) |
| 226 | |
| 227 | if i == 0 and role != "user": |
| 228 | # If the first message is from assistant (gpt), skip it. |
| 229 | # this happens when the conversation is a follow-up |
| 230 | # to a previous conversation (from the same user). |
| 231 | continue |
| 232 | |
| 233 | if max_turns is not None and i >= max_turns: |
| 234 | break |
| 235 | |
| 236 | # Convert message to OpenAI format (with "role" and "content") |
| 237 | content = turn["value"] |
| 238 | messages.append({"role": role, "content": content}) |
| 239 | |
| 240 | # Add the converted conversation to the OpenAI format |
| 241 | if len(messages) > 0: |
| 242 | valid_messages = True |
| 243 | |
| 244 | # First turn should always be from the user |
| 245 | user_turn = True |
| 246 | |
| 247 | for m in messages: |
| 248 | # Make sure that turns alternate between user and assistant |
| 249 | if (user_turn and m["role"] != "user") or ( |
| 250 | not user_turn and m["role"] != "assistant" |
| 251 | ): |
| 252 | valid_messages = False |
| 253 | break |
| 254 | |
| 255 | user_turn = not user_turn |
| 256 | |
| 257 | content = m["content"] |
| 258 | valid_messages = content_is_valid( |
| 259 | content, min_content_len, max_content_len |
| 260 | ) |
| 261 | if not valid_messages: |
| 262 | break |
| 263 | |
| 264 | if valid_messages is True: |
| 265 | final_openai_dataset.append({"id": conv_id, "messages": messages}) |
| 266 | |
| 267 | assert len(final_openai_dataset) > 0, "Final number of conversations is zero" |
| 268 | |
| 269 | print_stats(final_openai_dataset) |
| 270 | |
| 271 | print_stats_again = False |
| 272 | if max_items is not None and len(final_openai_dataset) > max_items: |
| 273 | print(f"\n\nSampling {max_items} items from the dataset...") |
| 274 | print_stats_again = True |
| 275 | final_openai_dataset = random.sample(final_openai_dataset, max_items) |
| 276 | |
| 277 | if print_stats_again: |
| 278 | # Print stats after the dataset changed |
| 279 | print_stats(final_openai_dataset, tokenizer) |
| 280 | |
| 281 | # Write the converted data to a new JSON file |
| 282 | final_size = len(final_openai_dataset) |
| 283 | print(f"\nTotal conversations converted (after filtering): {final_size:,}") |
| 284 | print(f"\nWriting file: {output_file}") |
| 285 | with open(output_file, "w", encoding="utf-8") as f: |
| 286 | json.dump(final_openai_dataset, f, ensure_ascii=False, indent=2) |
| 287 | |
| 288 | |
| 289 | def main() -> None: |
| 290 | parser = argparse.ArgumentParser( |
| 291 | description="Convert ShareGPT dataset to OpenAI API format" |
| 292 | ) |
| 293 | parser.add_argument("input_file", help="Path to the input ShareGPT JSON file") |
| 294 | parser.add_argument( |
| 295 | "output_file", help="Path to the output OpenAI format JSON file" |
| 296 | ) |
| 297 | parser.add_argument( |
| 298 | "--seed", type=int, default=0, help="Seed for random number generators" |
| 299 | ) |
| 300 | parser.add_argument( |
| 301 | "--max-items", |
| 302 | type=int, |
| 303 | default=None, |
| 304 | help="Maximum number of items in the output file", |
| 305 | ) |
| 306 | parser.add_argument( |
| 307 | "--min-turns", |
| 308 | type=int, |
| 309 | default=None, |
| 310 | help="Minimum number of turns per conversation", |
| 311 | ) |
| 312 | parser.add_argument( |
| 313 | "--max-turns", |
| 314 | type=int, |
| 315 | default=None, |
| 316 | help="Maximum number of turns per conversation", |
| 317 | ) |
| 318 | parser.add_argument( |
| 319 | "--min-content-len", |
| 320 | type=int, |
| 321 | default=None, |
| 322 | help="Min number of characters in the messages' content", |
| 323 | ) |
| 324 | parser.add_argument( |
| 325 | "--max-content-len", |
| 326 | type=int, |
| 327 | default=None, |
| 328 | help="Max number of characters in the messages' content", |
| 329 | ) |
| 330 | parser.add_argument( |
| 331 | "--model", |
| 332 | type=str, |
| 333 | default=None, |
| 334 | help="LLM model, only the tokenizer will be used", |
| 335 | ) |
| 336 | |
| 337 | args = parser.parse_args() |
| 338 | |
| 339 | convert_sharegpt_to_openai( |
| 340 | args.seed, |
| 341 | args.input_file, |
| 342 | args.output_file, |
| 343 | args.max_items, |
| 344 | args.min_content_len, |
| 345 | args.max_content_len, |
| 346 | args.min_turns, |
| 347 | args.max_turns, |
| 348 | args.model, |
| 349 | ) |
| 350 | |
| 351 | |
| 352 | if __name__ == "__main__": |
| 353 | main() |