@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 157 lines 6.18 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off - compares the generated bundle directly with its checked-in bytes.
2 import * as NodeFS from "node:fs";
3
4 import { catalog } from "@handy-computer/models";
5 import { recommendSpeechModels } from "@t3tools/shared/speech";
6 import policy from "../../../packages/shared/src/speechRecommendationPolicy.json" with { type: "json" };
7 import { expect, it } from "vite-plus/test";
8
9 import manifest from "../package.json" with { type: "json" };
10 import { generateSpeechCatalog, renderSpeechCatalog } from "./speech-catalog.ts";
11
12 const generate = (models = catalog.models) =>
13 generateSpeechCatalog(
14 { ...catalog.data, models },
15 manifest.devDependencies["@handy-computer/models"],
16 manifest.dependencies["transcribe-cpp"],
17 );
18
19 it("reproduces the bundled registry byte for byte without upstream requests", async () => {
20 const { fastCpuMaxWaitSeconds, ...methodology } = catalog.methodology;
21 expect(policy).toEqual({ ...methodology, fastMaxWaitSeconds: fastCpuMaxWaitSeconds });
22 expect(NodeFS.readFileSync(new URL("../src/speech/catalog.json", import.meta.url), "utf8")).toBe(
23 await renderSpeechCatalog(
24 catalog.data,
25 manifest.devDependencies["@handy-computer/models"],
26 manifest.dependencies["transcribe-cpp"],
27 ),
28 );
29 for (const model of generate().models) {
30 const source = catalog.models.find((entry) => entry.repository === model.id)!;
31 expect(model).toMatchObject({
32 revision: source.revision,
33 name: source.name,
34 ...source.files[source.quant],
35 supportsStreaming: source.capabilities.streaming,
36 supportsLanguageDetection: source.capabilities.languageDetection,
37 supportsTranslation: source.capabilities.translate,
38 });
39 }
40 const parakeet = catalog.models[0]!;
41 const withoutFrenchOrSpeed = generateSpeechCatalog(
42 {
43 ...catalog.data,
44 models: [{ ...parakeet, languages: ["en", "fr"] }],
45 results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance: {} } },
46 },
47 "0.2.3",
48 "0.4.1",
49 );
50 expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
51 accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
52 secondsPer30s: { cpu: null, accelerated: null },
53 });
54 expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2);
55 for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) {
56 const result = generateSpeechCatalog(
57 {
58 ...catalog.data,
59 models: [parakeet],
60 results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance } },
61 },
62 "0.2.3",
63 "0.4.1",
64 );
65 expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
66 cpu: performance.cpu ? 30 / performance.cpu : null,
67 accelerated: performance.accelerated ? 30 / performance.accelerated : null,
68 });
69 }
70 });
71
72 it("preserves Handy's accelerated balanced pick after filtering T3's offered candidates", () => {
73 const offered = generate().models;
74 const sources = catalog.models.filter((model) =>
75 offered.some(({ id }) => id === model.repository),
76 );
77 for (const languages of [["en"], ["de"], ["ja"], ["en", "de"], ["en", "fr"], ["hi", "mr"]]) {
78 const expected = catalog
79 .recommendModels(sources, languages)
80 .find((pick) => pick.status !== "unbenchmarked" && pick.roles.includes("best"));
81 expect(
82 recommendSpeechModels(offered, languages, "accelerated").keys().next().value,
83 languages.join(","),
84 ).toBe(expected?.model.repository);
85 }
86 const excluded = { ...catalog.models[0]!, license: "cc-by-nc-4.0" };
87 const allowed = { ...catalog.models[1]!, languages: ["en"] };
88 const filtered = generate([excluded, allowed]).models;
89 expect([...recommendSpeechModels(filtered, ["en"], "accelerated").keys()]).toEqual([
90 allowed.repository,
91 ]);
92 });
93
94 it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {
95 const model = catalog.models[0]!;
96 for (const override of [
97 { role: "langid" as const },
98 { role: "vad" as const },
99 { role: "diarize" as const },
100 { license: "cc-by-nc-4.0" },
101 { license: "CC-BY-NC-SA-4.0" },
102 { license: "cc-by-nc-nd-3.0" },
103 { license: "cc-by-nc" },
104 { license: "CC BY-NC 4.0" },
105 { license: "custom_NC" },
106 { minEngine: "0.4.2" },
107 { minEngine: "0.5.0" },
108 { family: "future_arch" },
109 ...[
110 "nested/model.gguf",
111 "nested\\model.gguf",
112 "../model.gguf",
113 "model.bin",
114 "model.safetensors",
115 ].map((filename) => ({ files: { [model.quant]: { ...model.files[model.quant]!, filename } } })),
116 ]) {
117 expect(generate([{ ...model, ...override }]).models).toEqual([]);
118 }
119 for (const license of ["other", "apache-2.0", "cc-by-4.0", "cc-by-sa-4.0", "unlicense"]) {
120 expect(generate([{ ...model, license, minEngine: "0.4.1" }]).models).toHaveLength(1);
121 }
122 const moonshine = catalog.models.find((entry) => entry.id === "moonshine-tiny-ja")!;
123 for (const [id, allowed] of [
124 ["moonshine-tiny-ja", false],
125 ["moonshine-base-vi", false],
126 ["moonshine-base-es", false],
127 ["moonshine-base", true],
128 ["moonshine-streaming-tiny-ja", true],
129 ] as const) {
130 const entry = { ...moonshine, id, repository: `handy-computer/${id}-gguf`, license: "mit" };
131 expect(generate([entry]).models, id).toHaveLength(allowed ? 1 : 0);
132 }
133 });
134
135 it("rejects missing defaults, unpinned or corrupt metadata, collisions, and unreviewed engine/schema changes", () => {
136 const model = catalog.models[0]!;
137 const file = model.files[model.quant]!;
138 for (const override of [
139 { files: {} },
140 { revision: "main" },
141 { repository: "../model" },
142 { languages: [] },
143 ...[{ sha256: "bad" }, { size: 0 }, { size: 1.5 }].map((entry) => ({
144 files: { [model.quant]: { ...file, ...entry } },
145 })),
146 ]) {
147 expect(() => generate([{ ...model, ...override }])).toThrow();
148 }
149 expect(() => generate([model, model])).toThrow(/Duplicate/);
150 expect(() => generate([model, { ...model, repository: "handy-computer/another" }])).toThrow(
151 /Duplicate/,
152 );
153 expect(() => generateSpeechCatalog(catalog.data, "0.2.3", "0.4.2")).toThrow(/loader support/);
154 expect(() =>
155 generateSpeechCatalog({ ...catalog.data, schema: 3 as 2 }, "0.2.3", "0.4.1"),
156 ).toThrow(/schema/);
157 });