The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off - compares the generated bundle directly with its checked-in bytes. |
| 2 | import * as NodeFS from "node:fs"; |
| 3 | |
| 4 | import { catalog } from "@handy-computer/models"; |
| 5 | import { recommendSpeechModels } from "@t3tools/shared/speech"; |
| 6 | import policy from "../../../packages/shared/src/speechRecommendationPolicy.json" with { type: "json" }; |
| 7 | import { expect, it } from "vite-plus/test"; |
| 8 | |
| 9 | import manifest from "../package.json" with { type: "json" }; |
| 10 | import { generateSpeechCatalog, renderSpeechCatalog } from "./speech-catalog.ts"; |
| 11 | |
| 12 | const generate = (models = catalog.models) => |
| 13 | generateSpeechCatalog( |
| 14 | { ...catalog.data, models }, |
| 15 | manifest.devDependencies["@handy-computer/models"], |
| 16 | manifest.dependencies["transcribe-cpp"], |
| 17 | ); |
| 18 | |
| 19 | it("reproduces the bundled registry byte for byte without upstream requests", async () => { |
| 20 | const { fastCpuMaxWaitSeconds, ...methodology } = catalog.methodology; |
| 21 | expect(policy).toEqual({ ...methodology, fastMaxWaitSeconds: fastCpuMaxWaitSeconds }); |
| 22 | expect(NodeFS.readFileSync(new URL("../src/speech/catalog.json", import.meta.url), "utf8")).toBe( |
| 23 | await renderSpeechCatalog( |
| 24 | catalog.data, |
| 25 | manifest.devDependencies["@handy-computer/models"], |
| 26 | manifest.dependencies["transcribe-cpp"], |
| 27 | ), |
| 28 | ); |
| 29 | for (const model of generate().models) { |
| 30 | const source = catalog.models.find((entry) => entry.repository === model.id)!; |
| 31 | expect(model).toMatchObject({ |
| 32 | revision: source.revision, |
| 33 | name: source.name, |
| 34 | ...source.files[source.quant], |
| 35 | supportsStreaming: source.capabilities.streaming, |
| 36 | supportsLanguageDetection: source.capabilities.languageDetection, |
| 37 | supportsTranslation: source.capabilities.translate, |
| 38 | }); |
| 39 | } |
| 40 | const parakeet = catalog.models[0]!; |
| 41 | const withoutFrenchOrSpeed = generateSpeechCatalog( |
| 42 | { |
| 43 | ...catalog.data, |
| 44 | models: [{ ...parakeet, languages: ["en", "fr"] }], |
| 45 | results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance: {} } }, |
| 46 | }, |
| 47 | "0.2.3", |
| 48 | "0.4.1", |
| 49 | ); |
| 50 | expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({ |
| 51 | accuracy: { en: { grade: "A-", errorPercent: 3.99 } }, |
| 52 | secondsPer30s: { cpu: null, accelerated: null }, |
| 53 | }); |
| 54 | expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2); |
| 55 | for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) { |
| 56 | const result = generateSpeechCatalog( |
| 57 | { |
| 58 | ...catalog.data, |
| 59 | models: [parakeet], |
| 60 | results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance } }, |
| 61 | }, |
| 62 | "0.2.3", |
| 63 | "0.4.1", |
| 64 | ); |
| 65 | expect(result.models[0]?.benchmark.secondsPer30s).toEqual({ |
| 66 | cpu: performance.cpu ? 30 / performance.cpu : null, |
| 67 | accelerated: performance.accelerated ? 30 / performance.accelerated : null, |
| 68 | }); |
| 69 | } |
| 70 | }); |
| 71 | |
| 72 | it("preserves Handy's accelerated balanced pick after filtering T3's offered candidates", () => { |
| 73 | const offered = generate().models; |
| 74 | const sources = catalog.models.filter((model) => |
| 75 | offered.some(({ id }) => id === model.repository), |
| 76 | ); |
| 77 | for (const languages of [["en"], ["de"], ["ja"], ["en", "de"], ["en", "fr"], ["hi", "mr"]]) { |
| 78 | const expected = catalog |
| 79 | .recommendModels(sources, languages) |
| 80 | .find((pick) => pick.status !== "unbenchmarked" && pick.roles.includes("best")); |
| 81 | expect( |
| 82 | recommendSpeechModels(offered, languages, "accelerated").keys().next().value, |
| 83 | languages.join(","), |
| 84 | ).toBe(expected?.model.repository); |
| 85 | } |
| 86 | const excluded = { ...catalog.models[0]!, license: "cc-by-nc-4.0" }; |
| 87 | const allowed = { ...catalog.models[1]!, languages: ["en"] }; |
| 88 | const filtered = generate([excluded, allowed]).models; |
| 89 | expect([...recommendSpeechModels(filtered, ["en"], "accelerated").keys()]).toEqual([ |
| 90 | allowed.repository, |
| 91 | ]); |
| 92 | }); |
| 93 | |
| 94 | it("filters restricted licenses, unsupported roles, engines, architectures and files", () => { |
| 95 | const model = catalog.models[0]!; |
| 96 | for (const override of [ |
| 97 | { role: "langid" as const }, |
| 98 | { role: "vad" as const }, |
| 99 | { role: "diarize" as const }, |
| 100 | { license: "cc-by-nc-4.0" }, |
| 101 | { license: "CC-BY-NC-SA-4.0" }, |
| 102 | { license: "cc-by-nc-nd-3.0" }, |
| 103 | { license: "cc-by-nc" }, |
| 104 | { license: "CC BY-NC 4.0" }, |
| 105 | { license: "custom_NC" }, |
| 106 | { minEngine: "0.4.2" }, |
| 107 | { minEngine: "0.5.0" }, |
| 108 | { family: "future_arch" }, |
| 109 | ...[ |
| 110 | "nested/model.gguf", |
| 111 | "nested\\model.gguf", |
| 112 | "../model.gguf", |
| 113 | "model.bin", |
| 114 | "model.safetensors", |
| 115 | ].map((filename) => ({ files: { [model.quant]: { ...model.files[model.quant]!, filename } } })), |
| 116 | ]) { |
| 117 | expect(generate([{ ...model, ...override }]).models).toEqual([]); |
| 118 | } |
| 119 | for (const license of ["other", "apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "unlicense"]) { |
| 120 | expect(generate([{ ...model, license, minEngine: "0.4.1" }]).models).toHaveLength(1); |
| 121 | } |
| 122 | const moonshine = catalog.models.find((entry) => entry.id === "moonshine-tiny-ja")!; |
| 123 | for (const [id, allowed] of [ |
| 124 | ["moonshine-tiny-ja", false], |
| 125 | ["moonshine-base-vi", false], |
| 126 | ["moonshine-base-es", false], |
| 127 | ["moonshine-base", true], |
| 128 | ["moonshine-streaming-tiny-ja", true], |
| 129 | ] as const) { |
| 130 | const entry = { ...moonshine, id, repository: `handy-computer/${id}-gguf`, license: "mit" }; |
| 131 | expect(generate([entry]).models, id).toHaveLength(allowed ? 1 : 0); |
| 132 | } |
| 133 | }); |
| 134 | |
| 135 | it("rejects missing defaults, unpinned or corrupt metadata, collisions, and unreviewed engine/schema changes", () => { |
| 136 | const model = catalog.models[0]!; |
| 137 | const file = model.files[model.quant]!; |
| 138 | for (const override of [ |
| 139 | { files: {} }, |
| 140 | { revision: "main" }, |
| 141 | { repository: "../model" }, |
| 142 | { languages: [] }, |
| 143 | ...[{ sha256: "bad" }, { size: 0 }, { size: 1.5 }].map((entry) => ({ |
| 144 | files: { [model.quant]: { ...file, ...entry } }, |
| 145 | })), |
| 146 | ]) { |
| 147 | expect(() => generate([{ ...model, ...override }])).toThrow(); |
| 148 | } |
| 149 | expect(() => generate([model, model])).toThrow(/Duplicate/); |
| 150 | expect(() => generate([model, { ...model, repository: "handy-computer/another" }])).toThrow( |
| 151 | /Duplicate/, |
| 152 | ); |
| 153 | expect(() => generateSpeechCatalog(catalog.data, "0.2.3", "0.4.2")).toThrow(/loader support/); |
| 154 | expect(() => |
| 155 | generateSpeechCatalog({ ...catalog.data, schema: 3 as 2 }, "0.2.3", "0.4.1"), |
| 156 | ).toThrow(/schema/); |
| 157 | }); |