[Docs] Add Qwen3 forced alignment online example (#46197)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Taneem Ibrahim committed
Jun 23, 2026 at 11:59 UTC
40e552212126717cf59dd493c2057dfdd5a1259e
1 file changed
+213
examples/pooling/token_classify/forced_alignment_online.py
new
+213
@@ -0,0 +1,213 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# Adapted from Qwen3-ForcedAligner inference:
4
+# https://github.com/QwenLM/Qwen3-ASR
5
+
6
+"""
7
+Online forced alignment example using Qwen3-ForcedAligner-0.6B.
8
+
9
+Forced alignment takes audio and reference text as input and produces
10
+word-level timestamps. The model predicts a time bin at each <timestamp>
11
+token position; multiplying by ``timestamp_segment_time`` gives milliseconds.
12
+
13
+Start the server with:
14
+
15
+ vllm serve Qwen/Qwen3-ForcedAligner-0.6B \\
16
+ --runner pooling \\
17
+ --enforce-eager \\
18
+ --trust-request-chat-template \\
19
+ --hf-overrides \\
20
+ '{"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]}'
21
+
22
+Then run:
23
+
24
+ python forced_alignment_online.py
25
+"""
26
+
27
+import argparse
28
+import json
29
+import mimetypes
30
+import wave
31
+from io import BytesIO
32
+from pathlib import Path
33
+from typing import Any
34
+
35
+import numpy as np
36
+import pybase64 as base64
37
+import requests
38
+import torch
39
+from huggingface_hub import hf_hub_download
40
+
41
+RAW_CONTENT_CHAT_TEMPLATE = "{{ messages[0]['content'] }}"
42
+
43
+
44
+def build_prompt(words: list[str]) -> str:
45
+ """Build the forced alignment prompt from a word list.
46
+
47
+ Format: <|audio_start|><|audio_pad|><|audio_end|>
48
+ word1<timestamp><timestamp>word2<timestamp><timestamp>...
49
+ """
50
+ body = "<timestamp><timestamp>".join(words) + "<timestamp><timestamp>"
51
+ return f"<|audio_start|><|audio_pad|><|audio_end|>{body}"
52
+
53
+
54
+def encode_audio_data_uri(audio_path: Path) -> str:
55
+ mime_type = mimetypes.guess_type(audio_path)[0] or "audio/wav"
56
+ audio_base64 = base64.b64encode(audio_path.read_bytes()).decode("utf-8")
57
+ return f"data:{mime_type};base64,{audio_base64}"
58
+
59
+
60
+def encode_silent_wav_data_uri(sample_rate: int = 16000, duration_s: int = 5) -> str:
61
+ audio = np.zeros(sample_rate * duration_s, dtype=np.int16)
62
+
63
+ with BytesIO() as audio_buffer:
64
+ with wave.open(audio_buffer, "wb") as wav_file:
65
+ wav_file.setnchannels(1)
66
+ wav_file.setsampwidth(np.dtype(np.int16).itemsize)
67
+ wav_file.setframerate(sample_rate)
68
+ wav_file.writeframes(audio.tobytes())
69
+
70
+ audio_base64 = base64.b64encode(audio_buffer.getvalue()).decode("utf-8")
71
+
72
+ return f"data:audio/wav;base64,{audio_base64}"
73
+
74
+
75
+def build_payload(model: str, prompt: str, audio_uri: str) -> dict[str, Any]:
76
+ return {
77
+ "model": model,
78
+ "messages": [
79
+ {
80
+ "role": "user",
81
+ "content": [
82
+ {"type": "text", "text": prompt},
83
+ {"type": "audio_url", "audio_url": {"url": audio_uri}},
84
+ ],
85
+ }
86
+ ],
87
+ "task": "token_classify",
88
+ "chat_template": RAW_CONTENT_CHAT_TEMPLATE,
89
+ }
90
+
91
+
92
+def post_http_request(payload: dict[str, Any], api_url: str) -> requests.Response:
93
+ headers = {"User-Agent": "Test Client"}
94
+ return requests.post(api_url, headers=headers, json=payload)
95
+
96
+
97
+def parse_response(response: requests.Response) -> dict[str, Any]:
98
+ try:
99
+ result = response.json()
100
+ except ValueError as exc:
101
+ raise RuntimeError(
102
+ f"Server returned non-JSON response: {response.text}"
103
+ ) from exc
104
+
105
+ if response.status_code != 200 or "data" not in result:
106
+ raise RuntimeError(f"Server error ({response.status_code}): {result}")
107
+
108
+ return result
109
+
110
+
111
+def load_timestamp_config(model: str) -> tuple[int, float]:
112
+ model_path = Path(model)
113
+ config_path = (
114
+ model_path / "config.json"
115
+ if model_path.exists()
116
+ else Path(hf_hub_download(repo_id=model, filename="config.json"))
117
+ )
118
+
119
+ with config_path.open() as f:
120
+ config = json.load(f)
121
+
122
+ return config["timestamp_token_id"], config["timestamp_segment_time"]
123
+
124
+
125
+def parse_args():
126
+ parser = argparse.ArgumentParser()
127
+ parser.add_argument("--host", type=str, default="localhost")
128
+ parser.add_argument("--port", type=int, default=8000)
129
+ parser.add_argument(
130
+ "--model",
131
+ type=str,
132
+ default="Qwen/Qwen3-ForcedAligner-0.6B",
133
+ )
134
+ parser.add_argument(
135
+ "--audio-path",
136
+ type=Path,
137
+ default=None,
138
+ help="Optional audio file. Defaults to a 5-second silent WAV.",
139
+ )
140
+ parser.add_argument(
141
+ "--words",
142
+ nargs="+",
143
+ default=["Hello", "world"],
144
+ help="Reference words to align against the audio.",
145
+ )
146
+ return parser.parse_args()
147
+
148
+
149
+def main(args):
150
+ from transformers import AutoTokenizer
151
+
152
+ api_url = f"http://{args.host}:{args.port}/pooling"
153
+ prompt = build_prompt(args.words)
154
+ audio_uri = (
155
+ encode_audio_data_uri(args.audio_path)
156
+ if args.audio_path
157
+ else encode_silent_wav_data_uri()
158
+ )
159
+ payload = build_payload(args.model, prompt, audio_uri)
160
+
161
+ pooling_response = post_http_request(payload=payload, api_url=api_url)
162
+ result = parse_response(pooling_response)
163
+
164
+ tokenizer = AutoTokenizer.from_pretrained(args.model)
165
+ timestamp_token_id, timestamp_segment_time = load_timestamp_config(args.model)
166
+
167
+ output = result["data"][0]
168
+ logits = torch.tensor(output["data"])
169
+ predictions = logits.argmax(dim=-1)
170
+ token_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"]
171
+ audio_pad_token_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
172
+
173
+ usage = result.get("usage") or {}
174
+ prompt_tokens = usage.get("prompt_tokens")
175
+ if prompt_tokens is not None and prompt_tokens != len(predictions):
176
+ raise RuntimeError(
177
+ "The response length does not match the reported prompt token count."
178
+ )
179
+
180
+ try:
181
+ audio_pad_index = token_ids.index(audio_pad_token_id)
182
+ except ValueError as exc:
183
+ raise RuntimeError("The prompt does not contain the audio pad token.") from exc
184
+
185
+ audio_token_shift = len(predictions) - len(token_ids)
186
+ if audio_token_shift < 0:
187
+ raise RuntimeError(
188
+ "The response is shorter than the locally tokenized prompt. "
189
+ "Check that the server was started with --trust-request-chat-template."
190
+ )
191
+
192
+ ts_predictions = []
193
+ for i, token_id in enumerate(token_ids):
194
+ if token_id != timestamp_token_id:
195
+ continue
196
+
197
+ prediction_index = i + audio_token_shift if i > audio_pad_index else i
198
+ ts_predictions.append(
199
+ predictions[prediction_index].item() * timestamp_segment_time
200
+ )
201
+
202
+ if len(ts_predictions) < len(args.words) * 2:
203
+ raise RuntimeError("The model did not return enough timestamp predictions.")
204
+
205
+ for i, word in enumerate(args.words):
206
+ start_ms = ts_predictions[i * 2]
207
+ end_ms = ts_predictions[i * 2 + 1]
208
+ print(f"{word:15s} {start_ms / 1000:.3f}s - {end_ms / 1000:.3f}s")
209
+
210
+
211
+if __name__ == "__main__":
212
+ args = parse_args()
213
+ main(args)