| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import asyncio |
| 4 | import json |
| 5 | |
| 6 | import httpx |
| 7 | from openai import OpenAI |
| 8 | |
| 9 | from vllm.assets.audio import AudioAsset |
| 10 | |
| 11 | |
| 12 | def sync_openai(audio_path: str, client: OpenAI, model: str): |
| 13 | with open(audio_path, "rb") as f: |
| 14 | translation = client.audio.translations.create( |
| 15 | file=f, |
| 16 | model=model, |
| 17 | response_format="json", |
| 18 | temperature=0.0, |
| 19 | # Additional params not provided by OpenAI API. |
| 20 | extra_body=dict( |
| 21 | language="it", |
| 22 | seed=4419, |
| 23 | repetition_penalty=1.3, |
| 24 | ), |
| 25 | ) |
| 26 | print("translation result:", translation.text) |
| 27 | |
| 28 | |
| 29 | async def stream_openai_response( |
| 30 | audio_path: str, base_url: str, api_key: str, model: str |
| 31 | ): |
| 32 | data = { |
| 33 | "language": "it", |
| 34 | "stream": True, |
| 35 | "model": model, |
| 36 | } |
| 37 | url = base_url + "/audio/translations" |
| 38 | headers = {"Authorization": f"Bearer {api_key}"} |
| 39 | print("translation result:", end=" ") |
| 40 | # OpenAI translation API client does not support streaming. |
| 41 | async with httpx.AsyncClient() as client: |
| 42 | with open(audio_path, "rb") as f: |
| 43 | async with client.stream( |
| 44 | "POST", url, files={"file": f}, data=data, headers=headers |
| 45 | ) as response: |
| 46 | async for line in response.aiter_lines(): |
| 47 | # Each line is a JSON object prefixed with 'data: ' |
| 48 | if line: |
| 49 | if line.startswith("data: "): |
| 50 | line = line[len("data: ") :] |
| 51 | # Last chunk, stream ends |
| 52 | if line.strip() == "[DONE]": |
| 53 | break |
| 54 | # Parse the JSON response |
| 55 | chunk = json.loads(line) |
| 56 | # Extract and print the content |
| 57 | content = chunk["choices"][0].get("delta", {}).get("content") |
| 58 | print(content, end="") |
| 59 | |
| 60 | |
| 61 | def main(): |
| 62 | foscolo = str(AudioAsset("azacinto_foscolo").get_local_path()) |
| 63 | |
| 64 | # Modify OpenAI's API key and API base to use vLLM's API server. |
| 65 | openai_api_key = "EMPTY" |
| 66 | openai_api_base = "http://localhost:8000/v1" |
| 67 | client = OpenAI( |
| 68 | api_key=openai_api_key, |
| 69 | base_url=openai_api_base, |
| 70 | ) |
| 71 | |
| 72 | model = client.models.list().data[0].id |
| 73 | print(f"Using model: {model}") |
| 74 | |
| 75 | sync_openai(foscolo, client, model) |
| 76 | # Run the asynchronous function |
| 77 | asyncio.run(stream_openai_response(foscolo, openai_api_base, openai_api_key, model)) |
| 78 | |
| 79 | |
| 80 | if __name__ == "__main__": |
| 81 | main() |