@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 73859fec58

[Frontend] Switch Python Harmony dependency to oss-harmony (#55128)

Signed-off-by: Anton Peganov <apeganov@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Anton Peganov committed Sep 27, 2026 at 22:43 UTC 73859fec5865700c5b2021b22890cb3b2005f66e
17 files changed +265 -56
.buildkite/intel_jobs/lm_eval_intel.yaml
+1 -2
@@ -23,8 +23,7 @@ steps:
23 commands:
24 - >-
25 bash .buildkite/scripts/hardware_ci/run-intel-test.sh
26 - 'uv pip install "gpt-oss[eval]==0.0.5" &&
27 - cd tests &&
26 + 'cd tests &&
27 pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
28 - label: LM Eval Large Models(2 GPUs)
29 key: lm-eval-large-models-2-gpus
.buildkite/test-amd.yaml
-2
@@ -1287,7 +1287,6 @@ steps:
1287 - vllm/platforms/rocm.py
1288 - tests/evals/gpt_oss/
1289 commands:
1290 - - uv pip install --system 'gpt-oss[eval]==0.0.5'
1290 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
1291
1292 - label: ":amd: (MI300) LM Eval Small Models Distributed"
@@ -3584,7 +3583,6 @@ steps:
3583 - vllm/_aiter_ops.py
3584 - vllm/platforms/rocm.py
3585 commands:
3587 - - uv pip install --system 'gpt-oss[eval]==0.0.5'
3586 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
3587
3588 - label: ":amd: (MI355 DPX) LM Eval Watermarking"
.buildkite/test_areas/lm_eval.yaml
-4
@@ -829,7 +829,6 @@ steps:
829 - vllm/model_executor/layers/quantization
830 - tests/evals/gpt_oss/
831 commands:
832 - - uv pip install --system 'gpt-oss[eval]==0.0.5'
832 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
833
834 - label: ":nvidia: (B200) GPQA Eval (GPT-OSS)"
@@ -843,7 +842,6 @@ steps:
842 - vllm/model_executor/layers/quantization
843 - tests/evals/gpt_oss/
844 commands:
846 - - uv pip install --system 'gpt-oss[eval]==0.0.5'
845 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
846 mirror:
847 amd:
@@ -865,7 +863,6 @@ steps:
863 - vllm/_aiter_ops.py
864 - vllm/platforms/rocm.py
865 commands:
868 - - uv pip install --system 'gpt-oss[eval]==0.0.5'
866 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
867
868 - label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
@@ -940,7 +937,6 @@ steps:
937 - vllm/model_executor/layers/quantization
938 - tests/evals/gpt_oss/
939 commands:
943 - - uv pip install --system 'gpt-oss[eval]==0.0.5'
940 - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
941
942 - label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload"
docker/Dockerfile
+4
@@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt
729 COPY requirements/test/cuda.in requirements/test/cuda.in
730 COPY requirements/test/cuda.txt requirements/test/cuda.txt
731 COPY requirements/dev.txt requirements/dev.txt
732 +# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
733 +COPY pyproject.toml pyproject.toml
734 COPY tools/use_existing_torch.py tools/use_existing_torch.py
735 COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
736 RUN --mount=type=cache,target=/opt/uv/cache \
@@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt
1145 COPY requirements/test/cuda.in requirements/test/cuda.in
1146 COPY requirements/test/cuda.txt requirements/test/cuda.txt
1147 COPY requirements/dev.txt requirements/dev.txt
1148 +# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
1149 +COPY pyproject.toml pyproject.toml
1150 COPY tools/use_existing_torch.py tools/use_existing_torch.py
1151 COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
1152 RUN --mount=type=cache,target=/opt/uv/cache \
pyproject.toml
+5
@@ -258,3 +258,8 @@ CANN = "CANN"
258
259 [tool.uv]
260 no-build-isolation-package = ["torch"]
261 +# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony`
262 +# module without downloading tiktoken vocabularies at runtime. Installing
263 +# openai-harmony alongside it silently overwrites those files, so keep it out
264 +# of every resolution; gpt-oss pulls it in transitively.
265 +exclude-dependencies = ["openai-harmony"]
requirements/common.txt
+1 -1
@@ -49,7 +49,7 @@ pybase64 # fast base64 implementation
49 cbor2 # Required for cross-language serialization of hashable objects
50 ijson # Required for mistral streaming tool parser
51 setproctitle # Used to set process names for better debugging and monitoring
52 -openai-harmony >= 0.0.3 # Required for gpt-oss
52 +oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies
53 anthropic >= 0.71.0
54 model-hosting-container-standards >= 0.1.14, < 1.0.0
55 mcp >= 2.0.0, < 3.0.0
requirements/test/cpu.txt
+31 -10
@@ -76,7 +76,9 @@ black==24.10.0
76 blake3==1.0.9
77 # via -r requirements/test/../common.txt
78 blobfile==3.0.0
79 - # via -r requirements/test/cuda.in
79 + # via
80 + # -r requirements/test/cuda.in
81 + # gpt-oss
82 bm25s==0.2.13
83 # via mteb
84 boto3==1.35.57
@@ -111,7 +113,7 @@ chardet==5.2.0
113 # via mbstrdecoder
114 charset-normalizer==3.4.0
115 # via requests
114 -chz==0.3.0
116 +chz==0.4.0
117 # via gpt-oss
118 click==8.4.2
119 # via
@@ -190,7 +192,7 @@ distro==1.9.0
192 # openai
193 dnspython==2.7.0
194 # via email-validator
193 -docker==7.1.0
195 +docker==7.2.0
196 # via gpt-oss
197 docopt==0.6.2
198 # via num2words
@@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0
293 # google-api-core
294 # opentelemetry-exporter-otlp-proto-grpc
295 # opentelemetry-exporter-otlp-proto-http
294 -gpt-oss==0.0.8
296 +gpt-oss==0.0.9
297 # via -r requirements/test/cuda.in
298 graphql-core==3.2.6
299 # via hypothesis-graphql
@@ -402,6 +404,7 @@ jinja2==3.1.6
404 # datamodel-code-generator
405 # fastapi
406 # genai-perf
407 + # gpt-oss
408 # lm-eval
409 # torch
410 jiter==0.15.0
@@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1
436 # via jsonschema
437 junit-xml==1.9
438 # via schemathesis
439 +jupyter-client==8.10.0
440 + # via gpt-oss
441 +jupyter-core==5.9.1
442 + # via jupyter-client
443 kaldi-native-fbank==1.22.3
444 # via -r requirements/test/cuda.in
445 kaleido==0.2.1
@@ -549,6 +556,7 @@ numpy==2.2.6
556 # evaluate
557 # fastparquet
558 # genai-perf
559 + # gpt-oss
560 # imagehash
561 # imageio
562 # librosa
@@ -583,8 +591,6 @@ numpy==2.2.6
591 open-clip-torch==2.32.0
592 # via -r requirements/test/cuda.in
593 openai==2.44.0
586 - # via -r requirements/test/../common.txt
587 -openai-harmony==0.0.4
594 # via
595 # -r requirements/test/../common.txt
596 # gpt-oss
@@ -641,6 +647,8 @@ optuna==3.6.1
647 # via genai-perf
648 orjson==3.11.5
649 # via genai-perf
650 +oss-harmony==0.0.11
651 + # via -r requirements/test/../common.txt
652 outlines-core==0.2.14
653 # via -r requirements/test/../common.txt
654 packaging==24.2
@@ -672,6 +680,7 @@ pandas==2.2.3
680 # evaluate
681 # fastparquet
682 # genai-perf
683 + # gpt-oss
684 # statsmodels
685 partial-json-parser==0.2.1.1.post7
686 # via -r requirements/test/../common.txt
@@ -702,6 +711,7 @@ pillow==10.4.0
711 platformdirs==4.3.6
712 # via
713 # black
714 + # jupyter-core
715 # pooch
716 # virtualenv
717 plotly==5.24.1
@@ -794,7 +804,7 @@ pydantic==2.12.0
804 # model-hosting-container-standards
805 # mteb
806 # openai
797 - # openai-harmony
807 + # oss-harmony
808 # pydantic-extra-types
809 # pydantic-settings
810 # ray
@@ -855,6 +865,7 @@ pytest-timeout==2.3.1
865 python-dateutil==2.9.0.post0
866 # via
867 # botocore
868 + # jupyter-client
869 # matplotlib
870 # pandas
871 # typepy
@@ -898,7 +909,9 @@ pyyaml==6.0.2
909 # uvicorn
910 # vocos
911 pyzmq==27.1.0
901 - # via -r requirements/test/../common.txt
912 + # via
913 + # -r requirements/test/../common.txt
914 + # jupyter-client
915 rapidfuzz==3.12.1
916 # via jiwer
917 ray==2.56.1
@@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1
1090 # via schemathesis
1091 statsmodels==0.14.4
1092 # via genai-perf
1080 -structlog==25.4.0
1093 +structlog==26.1.0
1094 # via gpt-oss
1095 supervisor==4.3.0
1096 # via model-hosting-container-standards
@@ -1101,7 +1114,7 @@ tenacity==9.1.2
1114 # schemathesis
1115 tensorizer==2.10.1
1116 # via -r requirements/test/cuda.in
1104 -termcolor==3.1.0
1117 +termcolor==3.3.0
1118 # via gpt-oss
1119 threadpoolctl==3.5.0
1120 # via scikit-learn
@@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu
1170 # open-clip-torch
1171 # segmentation-models-pytorch
1172 # timm
1173 +tornado==6.5.10
1174 + # via jupyter-client
1175 tqdm==4.67.3
1176 # via
1177 # -r requirements/test/../common.txt
1178 # datasets
1179 # evaluate
1180 + # gpt-oss
1181 # huggingface-hub
1182 # lm-eval
1183 # mteb
@@ -1174,6 +1190,10 @@ tqdm==4.67.3
1190 # segmentation-models-pytorch
1191 # sentence-transformers
1192 # transformers
1193 +traitlets==5.16.1
1194 + # via
1195 + # jupyter-client
1196 + # jupyter-core
1197 transformers==5.17.0
1198 # via
1199 # -r requirements/test/../common.txt
@@ -1226,6 +1246,7 @@ typing-extensions==4.15.0
1246 # grpcio
1247 # httpx2
1248 # huggingface-hub
1249 + # jupyter-client
1250 # librosa
1251 # lm-eval
1252 # mcp
requirements/test/cuda.in
+1 -1
@@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64"
66 imagehash # Required for Prithvi tests
67 segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests
68
69 -gpt-oss >= 0.0.7; python_version > '3.11'
69 +gpt-oss[eval] >= 0.0.7; python_version > '3.11'
70
71 perceptron # required for isaac test
72 kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
requirements/test/cuda.txt
+31 -11
@@ -81,7 +81,9 @@ black==24.10.0
81 blake3==1.0.9
82 # via -r requirements/test/../common.txt
83 blobfile==3.0.0
84 - # via -r requirements/test/cuda.in
84 + # via
85 + # -r requirements/test/cuda.in
86 + # gpt-oss
87 bm25s==0.2.13
88 # via mteb
89 boto3==1.35.57
@@ -116,7 +118,7 @@ chardet==5.2.0
118 # via mbstrdecoder
119 charset-normalizer==3.4.0
120 # via requests
119 -chz==0.3.0
121 +chz==0.4.0
122 # via gpt-oss
123 click==8.4.2
124 # via
@@ -205,7 +207,7 @@ distro==1.9.0
207 # openai
208 dnspython==2.7.0
209 # via email-validator
208 -docker==7.1.0
210 +docker==7.2.0
211 # via gpt-oss
212 docopt==0.6.2
213 # via num2words
@@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0
312 # google-api-core
313 # opentelemetry-exporter-otlp-proto-grpc
314 # opentelemetry-exporter-otlp-proto-http
313 -gpt-oss==0.0.8
315 +gpt-oss==0.0.9
316 # via -r requirements/test/cuda.in
317 graphql-core==3.2.6
318 # via hypothesis-graphql
@@ -424,6 +426,7 @@ jinja2==3.1.6
426 # datamodel-code-generator
427 # fastapi
428 # genai-perf
429 + # gpt-oss
430 # lm-eval
431 # torch
432 jiter==0.15.0
@@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1
459 # via jsonschema
460 junit-xml==1.9
461 # via schemathesis
462 +jupyter-client==8.10.0
463 + # via gpt-oss
464 +jupyter-core==5.9.1
465 + # via jupyter-client
466 kaldi-native-fbank==1.22.3
467 # via -r requirements/test/cuda.in
468 kaleido==0.2.1
@@ -584,6 +591,7 @@ numpy==2.2.6
591 # evaluate
592 # fastparquet
593 # genai-perf
594 + # gpt-oss
595 # imagehash
596 # imageio
597 # librosa
@@ -662,10 +670,6 @@ openai==2.44.0
670 # via
671 # -c requirements/common.txt
672 # -r requirements/test/../common.txt
665 -openai-harmony==0.0.4
666 - # via
667 - # -c requirements/common.txt
668 - # -r requirements/test/../common.txt
673 # gpt-oss
674 opencensus==0.11.4
675 # via ray
@@ -727,6 +731,10 @@ optuna==3.6.1
731 # via genai-perf
732 orjson==3.11.5
733 # via genai-perf
734 +oss-harmony==0.0.11
735 + # via
736 + # -c requirements/common.txt
737 + # -r requirements/test/../common.txt
738 outlines-core==0.2.14
739 # via
740 # -c requirements/common.txt
@@ -760,6 +768,7 @@ pandas==2.2.3
768 # evaluate
769 # fastparquet
770 # genai-perf
771 + # gpt-oss
772 # statsmodels
773 partial-json-parser==0.2.1.1.post7
774 # via -r requirements/test/../common.txt
@@ -790,6 +799,7 @@ pillow==10.4.0
799 platformdirs==4.3.6
800 # via
801 # black
802 + # jupyter-core
803 # pooch
804 # virtualenv
805 plotly==5.24.1
@@ -887,7 +897,7 @@ pydantic==2.12.0
897 # model-hosting-container-standards
898 # mteb
899 # openai
890 - # openai-harmony
900 + # oss-harmony
901 # pydantic-extra-types
902 # pydantic-settings
903 # ray
@@ -948,6 +958,7 @@ pytest-timeout==2.3.1
958 python-dateutil==2.9.0.post0
959 # via
960 # botocore
961 + # jupyter-client
962 # matplotlib
963 # pandas
964 # typepy
@@ -994,6 +1005,7 @@ pyzmq==27.1.0
1005 # via
1006 # -c requirements/common.txt
1007 # -r requirements/test/../common.txt
1008 + # jupyter-client
1009 rapidfuzz==3.12.1
1010 # via jiwer
1011 ray==2.56.1
@@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1
1189 # via schemathesis
1190 statsmodels==0.14.4
1191 # via genai-perf
1180 -structlog==25.4.0
1192 +structlog==26.1.0
1193 # via gpt-oss
1194 supervisor==4.3.0
1195 # via model-hosting-container-standards
@@ -1201,7 +1213,7 @@ tenacity==9.1.2
1213 # schemathesis
1214 tensorizer==2.10.1
1215 # via -r requirements/test/cuda.in
1204 -termcolor==3.1.0
1216 +termcolor==3.3.0
1217 # via gpt-oss
1218 threadpoolctl==3.5.0
1219 # via scikit-learn
@@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130
1276 # open-clip-torch
1277 # segmentation-models-pytorch
1278 # timm
1279 +tornado==6.5.10
1280 + # via jupyter-client
1281 tqdm==4.67.3
1282 # via
1283 # -r requirements/test/../common.txt
1284 # datasets
1285 # evaluate
1286 + # gpt-oss
1287 # huggingface-hub
1288 # lm-eval
1289 # mteb
@@ -1281,6 +1296,10 @@ tqdm==4.67.3
1296 # segmentation-models-pytorch
1297 # sentence-transformers
1298 # transformers
1299 +traitlets==5.16.1
1300 + # via
1301 + # jupyter-client
1302 + # jupyter-core
1303 transformers==5.17.0
1304 # via
1305 # -c requirements/common.txt
@@ -1337,6 +1356,7 @@ typing-extensions==4.15.0
1356 # grpcio
1357 # httpx2
1358 # huggingface-hub
1359 + # jupyter-client
1360 # librosa
1361 # lm-eval
1362 # mcp
requirements/test/rocm.in
+1 -1
@@ -65,7 +65,7 @@ decord==0.6.0
65 imagehash # Required for Prithvi tests
66 segmentation-models-pytorch>0.4.0 # Required for Prithvi tests
67
68 -gpt-oss>=0.0.7; python_version > '3.11'
68 +gpt-oss[eval]>=0.0.7; python_version > '3.11'
69
70 perceptron # required for isaac test
71 kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
requirements/test/rocm.txt
+29 -9
@@ -80,7 +80,9 @@ black==26.3.1
80 blake3==1.0.8
81 # via -r requirements/test/../common.txt
82 blobfile==3.0.0
83 - # via -r requirements/test/rocm.in
83 + # via
84 + # -r requirements/test/rocm.in
85 + # gpt-oss
86 bm25s==0.2.13
87 # via mteb
88 boto3==1.42.74
@@ -198,7 +200,7 @@ distro==1.9.0
200 # openai
201 dnspython==2.8.0
202 # via email-validator
201 -docker==7.1.0
203 +docker==7.2.0
204 # via gpt-oss
205 docopt==0.6.2
206 # via num2words
@@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0
302 # google-api-core
303 # opentelemetry-exporter-otlp-proto-grpc
304 # opentelemetry-exporter-otlp-proto-http
303 -gpt-oss==0.0.8
305 +gpt-oss==0.0.9
306 # via -r requirements/test/rocm.in
307 graphql-core==3.2.8
308 # via hypothesis-graphql
@@ -415,6 +417,7 @@ jinja2==3.1.6
417 # datamodel-code-generator
418 # fastapi
419 # genai-perf
420 + # gpt-oss
421 # lm-eval
422 # torch
423 jiter==0.14.0
@@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1
450 # via jsonschema
451 junit-xml==1.9
452 # via schemathesis
453 +jupyter-client==8.10.0
454 + # via gpt-oss
455 +jupyter-core==5.9.1
456 + # via jupyter-client
457 kaldi-native-fbank==1.22.3
458 # via -r requirements/test/rocm.in
459 kaleido==1.0.0
@@ -587,6 +594,7 @@ numpy==2.2.6
594 # evaluate
595 # fastparquet
596 # genai-perf
597 + # gpt-oss
598 # imagehash
599 # imageio
600 # librosa
@@ -629,10 +637,6 @@ openai==2.31.0
637 # via
638 # -c requirements/common.txt
639 # -r requirements/test/../common.txt
632 -openai-harmony==0.0.8
633 - # via
634 - # -c requirements/common.txt
635 - # -r requirements/test/../common.txt
640 # gpt-oss
641 opencensus==0.11.4
642 # via ray
@@ -699,6 +703,10 @@ orjson==3.11.7
703 # via
704 # genai-perf
705 # kaleido
706 +oss-harmony==0.0.11
707 + # via
708 + # -c requirements/common.txt
709 + # -r requirements/test/../common.txt
710 outlines-core==0.2.14
711 # via
712 # -c requirements/common.txt
@@ -734,6 +742,7 @@ pandas==3.0.1
742 # evaluate
743 # fastparquet
744 # genai-perf
745 + # gpt-oss
746 # statsmodels
747 partial-json-parser==0.2.1.1.post7
748 # via -r requirements/test/../common.txt
@@ -764,6 +773,7 @@ pillow==12.1.1
773 platformdirs==4.3.6
774 # via
775 # black
776 + # jupyter-core
777 # pooch
778 # python-discovery
779 # virtualenv
@@ -861,7 +871,7 @@ pydantic==2.12.5
871 # model-hosting-container-standards
872 # mteb
873 # openai
864 - # openai-harmony
874 + # oss-harmony
875 # pydantic-extra-types
876 # pydantic-settings
877 # ray
@@ -922,6 +932,7 @@ pytest-timeout==2.3.1
932 python-dateutil==2.9.0.post0
933 # via
934 # botocore
935 + # jupyter-client
936 # matplotlib
937 # pandas
938 # typepy
@@ -970,6 +981,7 @@ pyzmq==27.1.0
981 # via
982 # -c requirements/common.txt
983 # -r requirements/test/../common.txt
984 + # jupyter-client
985 rapidfuzz==3.12.1
986 # via
987 # -r requirements/test/rocm.in
@@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1
1171 # via schemathesis
1172 statsmodels==0.14.6
1173 # via genai-perf
1162 -structlog==25.5.0
1174 +structlog==26.1.0
1175 # via gpt-oss
1176 supervisor==4.3.0
1177 # via model-hosting-container-standards
@@ -1215,11 +1227,14 @@ tokenizers==0.23.1
1227 # transformers
1228 torch-c-dlpack-ext==0.1.5
1229 # via tilelang
1230 +tornado==6.5.10
1231 + # via jupyter-client
1232 tqdm==4.67.3
1233 # via
1234 # -r requirements/test/../common.txt
1235 # datasets
1236 # evaluate
1237 + # gpt-oss
1238 # huggingface-hub
1239 # lm-eval
1240 # mteb
@@ -1233,6 +1248,10 @@ tqdm==4.67.3
1248 # sentence-transformers
1249 # tilelang
1250 # transformers
1251 +traitlets==5.16.1
1252 + # via
1253 + # jupyter-client
1254 + # jupyter-core
1255 transformers==5.17.0
1256 # via
1257 # -c requirements/common.txt
@@ -1287,6 +1306,7 @@ typing-extensions==4.15.0
1306 # grpcio
1307 # httpx2
1308 # huggingface-hub
1309 + # jupyter-client
1310 # librosa
1311 # lm-eval
1312 # mcp
requirements/test/xpu.in
+1 -1
@@ -32,7 +32,7 @@ soundfile
32 # --- Tool Parsing & Evaluation ---
33 blobfile
34 rapidfuzz
35 -gpt-oss >= 0.0.7; python_version > '3.11'
35 +gpt-oss[eval] >= 0.0.7; python_version > '3.11'
36 schemathesis>=4.0.0 # Required for openai schema test.
37 jiwer
38 bm25s
requirements/test/xpu.txt
+29 -8
@@ -67,7 +67,9 @@ azure-storage-blob==12.30.1
67 blake3==1.0.9
68 # via -r requirements/test/../common.txt
69 blobfile==3.2.0
70 - # via -r requirements/test/xpu.in
70 + # via
71 + # -r requirements/test/xpu.in
72 + # gpt-oss
73 bm25s==0.3.10
74 # via
75 # -r requirements/test/xpu.in
@@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1
259 # google-api-core
260 # opentelemetry-exporter-otlp-proto-grpc
261 # opentelemetry-exporter-otlp-proto-http
260 -gpt-oss==0.0.8
262 +gpt-oss==0.0.9
263 # via -r requirements/test/xpu.in
264 graphql-core==3.2.11
265 # via hypothesis-graphql
@@ -377,6 +379,7 @@ jinja2==3.1.6
379 # via
380 # -c requirements/xpu.txt
381 # fastapi
382 + # gpt-oss
383 # lm-eval
384 # torch
385 jiter==0.16.0
@@ -406,6 +409,10 @@ jsonschema-rs==0.49.9
409 # via schemathesis
410 jsonschema-specifications==2025.9.1
411 # via jsonschema
412 +jupyter-client==8.10.0
413 + # via gpt-oss
414 +jupyter-core==5.9.1
415 + # via jupyter-client
416 kiwisolver==1.5.1
417 # via matplotlib
418 lark==1.2.2
@@ -526,6 +533,7 @@ numpy==2.2.6
533 # datasets
534 # decord
535 # evaluate
536 + # gpt-oss
537 # librosa
538 # lm-eval
539 # matplotlib
@@ -575,10 +583,6 @@ openai==3.2.0
583 # via
584 # -c requirements/common.txt
585 # -r requirements/test/../common.txt
578 -openai-harmony==0.0.8
579 - # via
580 - # -c requirements/common.txt
581 - # -r requirements/test/../common.txt
586 # gpt-oss
587 opencv-python-headless==5.0.0.93
588 # via
@@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1
636 # -c requirements/common.txt
637 # -r requirements/test/../common.txt
638 # -r requirements/test/xpu.in
639 +oss-harmony==0.0.11
640 + # via
641 + # -c requirements/common.txt
642 + # -r requirements/test/../common.txt
643 outlines-core==0.2.14
644 # via
645 # -c requirements/common.txt
@@ -656,6 +664,7 @@ pandas==3.0.5
664 # via
665 # datasets
666 # evaluate
667 + # gpt-oss
668 partial-json-parser==0.2.1.1.post7
669 # via -r requirements/test/../common.txt
670 pathvalidate==3.3.1
@@ -667,7 +676,9 @@ pillow==12.3.0
676 # mistral-common
677 # torchvision
678 platformdirs==4.11.3
670 - # via pooch
679 + # via
680 + # jupyter-core
681 + # pooch
682 pluggy==1.6.0
683 # via
684 # pytest
@@ -747,7 +758,7 @@ pydantic==2.13.4
758 # model-hosting-container-standards
759 # mteb
760 # openai
750 - # openai-harmony
761 + # oss-harmony
762 # pydantic-extra-types
763 # pydantic-settings
764 # xgrammar
@@ -806,6 +817,7 @@ pytest-timeout==2.4.0
817 python-dateutil==2.9.0.post0
818 # via
819 # botocore
820 + # jupyter-client
821 # matplotlib
822 # pandas
823 # typepy
@@ -841,6 +853,7 @@ pyzmq==27.1.0
853 # via
854 # -c requirements/common.txt
855 # -r requirements/test/../common.txt
856 + # jupyter-client
857 rapidfuzz==3.14.5
858 # via
859 # -r requirements/test/xpu.in
@@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu
1079 # via
1080 # open-clip-torch
1081 # timm
1082 +tornado==6.5.10
1083 + # via jupyter-client
1084 tqdm==4.70.0
1085 # via
1086 # -r requirements/test/../common.txt
1087 # datasets
1088 # evaluate
1089 + # gpt-oss
1090 # huggingface-hub
1091 # lm-eval
1092 # modelscope
@@ -1081,6 +1097,10 @@ tqdm==4.70.0
1097 # pqdm
1098 # sentence-transformers
1099 # transformers
1100 +traitlets==5.16.1
1101 + # via
1102 + # jupyter-client
1103 + # jupyter-core
1104 transformers==5.17.0
1105 # via
1106 # -c requirements/common.txt
@@ -1132,6 +1152,7 @@ typing-extensions==4.16.0
1152 # grpcio
1153 # httpx2
1154 # huggingface-hub
1155 + # jupyter-client
1156 # lm-eval
1157 # mcp
1158 # mcp-types
tests/entrypoints/openai/parser/test_harmony_render_parity.py
+18
@@ -20,6 +20,7 @@ Each test:
20 """
21
22 from openai.types.responses import ResponseFunctionToolCall
23 +from openai_harmony import Message, Role
24
25 from tests.entrypoints.openai.utils import verify_harmony_messages
26 from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import (
34 response_previous_input_to_harmony,
35 )
36
37 +
38 +def test_render_uses_legacy_tool_call_header():
39 + tool_call = (
40 + Message.from_role_and_content(Role.ASSISTANT, "print('hello')")
41 + .with_channel("commentary")
42 + .with_recipient("python")
43 + .with_content_type("code")
44 + )
45 +
46 + rendered = get_encoding().decode(render_for_completion([tool_call]))
47 +
48 + assert (
49 + "<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered
50 + )
51 + assert "commentary to=python <|constrain|>code" not in rendered
52 +
53 +
54 # Use a fixed date so the system message is deterministic across both paths.
55 _DATE = "2025-01-01"
56
tests/evals/gpt_oss/run_gpqa_eval.py new
+34
@@ -0,0 +1,34 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Run the gpt-oss GPQA evaluator with an explicit output-token budget."""
4 +
5 +import argparse
6 +
7 +from gpt_oss.evals.gpqa_eval import GPQAEval
8 +from gpt_oss.evals.responses_sampler import ResponsesSampler
9 +
10 +
11 +def main() -> None:
12 + parser = argparse.ArgumentParser()
13 + parser.add_argument("--model", required=True)
14 + parser.add_argument("--reasoning-effort", required=True)
15 + parser.add_argument("--base-url", required=True)
16 + parser.add_argument("--n-threads", required=True, type=int)
17 + parser.add_argument("--max-output-tokens", required=True, type=int)
18 + args = parser.parse_args()
19 +
20 + sampler = ResponsesSampler(
21 + model=args.model,
22 + reasoning_model=True,
23 + reasoning_effort=args.reasoning_effort,
24 + base_url=args.base_url,
25 + max_tokens=args.max_output_tokens,
26 + )
27 + result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler)
28 + # gpt-oss returns a NumPy scalar. Convert it so the parent process gets a
29 + # stable, dependency-independent representation that its regex can parse.
30 + print({"metric": float(result.score)})
31 +
32 +
33 +if __name__ == "__main__":
34 + main()
tests/evals/gpt_oss/test_gpqa_correctness.py
+9 -4
@@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer
21
22 TOL = 0.05 # Absolute tolerance for accuracy comparison
23
24 +# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output
25 +# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context.
26 +# This test originally ran with a 32768-token model context, so retain that
27 +# proven generation budget while using the current evaluator package.
28 +MAX_OUTPUT_TOKENS = 32_768
29 +
30 # Path to tiktoken encoding files
31 TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
32
@@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
56 # Build the command to run the evaluation
57 cmd = [
58 sys.executable,
53 - "-m",
54 - "gpt_oss.evals",
55 - "--eval",
56 - "gpqa",
59 + str(Path(__file__).with_name("run_gpqa_eval.py")),
60 "--model",
61 model_name,
62 "--reasoning-effort",
@@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
65 base_url,
66 "--n-threads",
67 "200",
68 + "--max-output-tokens",
69 + str(MAX_OUTPUT_TOKENS),
70 ]
71
72 try:
vllm/entrypoints/openai/parser/harmony_utils.py
+70 -2
@@ -449,12 +449,80 @@ def parse_chat_input_to_harmony_message(
449 def render_for_completion(messages: list[Message]) -> list[int]:
450 messages = auto_drop_analysis_messages(messages)
451 conversation = Conversation.from_messages(messages)
452 - token_ids = get_encoding().render_conversation_for_completion(
452 + encoding = get_encoding()
453 + token_ids = encoding.render_conversation_for_completion(
454 conversation,
455 Role.ASSISTANT,
456 config=RenderConversationConfig(auto_drop_analysis=False),
457 )
457 - return token_ids
458 + return _use_legacy_tool_call_headers(token_ids)
459 +
460 +
461 +def _use_legacy_tool_call_headers(token_ids: list[int]) -> list[int]:
462 + """Keep prior assistant tool calls in the format expected by GPT-OSS.
463 +
464 + oss-harmony 0.0.10 changed a rendered tool-call header from::
465 +
466 + assistant to=python<|channel|>commentary code
467 +
468 + to::
469 +
470 + assistant<|channel|>commentary to=python <|constrain|>code
471 +
472 + GPT-OSS can echo the latter as a malformed header with the recipient in
473 + both positions. Rewrite only that unambiguous token-level header shape,
474 + while retaining oss-harmony's embedded vocabulary and parser.
475 + """
476 + encoding = get_encoding()
477 + start = encoding.encode("<|start|>", allowed_special="all")[0]
478 + channel_token = encoding.encode("<|channel|>", allowed_special="all")[0]
479 + constrain = encoding.encode("<|constrain|>", allowed_special="all")[0]
480 + message = encoding.encode("<|message|>", allowed_special="all")[0]
481 +
482 + result = token_ids.copy()
483 + index = 0
484 + while index < len(result):
485 + if result[index] != start:
486 + index += 1
487 + continue
488 +
489 + try:
490 + header_end = result.index(message, index + 1)
491 + channel_index = result.index(channel_token, index + 1, header_end)
492 + constrain_index = result.index(constrain, channel_index + 1, header_end)
493 + except ValueError:
494 + index += 1
495 + continue
496 +
497 + author = encoding.decode(result[index + 1 : channel_index])
498 + channel_and_recipient = encoding.decode(
499 + result[channel_index + 1 : constrain_index]
500 + )
501 + content_type = encoding.decode(result[constrain_index + 1 : header_end])
502 + if (
503 + author != "assistant"
504 + or " to=" not in channel_and_recipient
505 + or not channel_and_recipient.endswith(" ")
506 + ):
507 + index = header_end + 1
508 + continue
509 +
510 + channel, recipient = channel_and_recipient[:-1].split(" to=", 1)
511 + if not channel or not recipient or not content_type:
512 + index = header_end + 1
513 + continue
514 +
515 + legacy_header = (
516 + [start]
517 + + encoding.encode(f"assistant to={recipient}")
518 + + [channel_token]
519 + + encoding.encode(f"{channel} {content_type}")
520 + + [message]
521 + )
522 + result[index : header_end + 1] = legacy_header
523 + index += len(legacy_header)
524 +
525 + return result
526
527
528 def get_streamable_parser_for_assistant() -> StreamableParser: