@devries-cameron20 / composio-1 / commits / 4f11a440e
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(cli): stop tiktoken special-token literals from failing execute

`Tiktoken.encode` signs as `encode(text, allowedSpecial = [], disallowedSpecial = "all")`. The CLI passed only the text, so every special token was disallowed and the call threw on any response containing the literal `<|endoftext|>` or `<|endofprompt|>`. A 66-byte payload is enough. Reading a README that documents a tokenizer hits it. The throw landed in `prepareExecuteOutput`, after `spinner.stop('Execution successful')` had already printed. So the tool had run, its side effects had happened, and the CLI still exited 1 with nothing on stdout and no session history entry. Here the encoder is only a length gauge for the inline-versus-file decision, so those literals are ordinary characters. Passing `allowedSpecial: 'all'` counts them instead of rejecting the payload. Token counts are unchanged on text that contains no special tokens. The preceding commit's byte-length pre-filter hid this below 10KB by skipping the encoder. Larger responses still reached it, and both call sites were affected: the threshold check and the `tokenCount` reported for a stored file. Both now go through one `countOutputTokens` helper. The regression test drives the real command with a response holding both literals, sized past the inline threshold so the count is actually computed. Verified it fails without the fix, with the original error: Error: The text contains a special token that is not allowed: <|endoftext|> Verified with typecheck (src and test), oxlint, validate:boundaries, and the tools.execute and run command suites, now 90 tests. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EzaE7oGVgziJ5nRvBhcci2

Claude committed Sep 12, 2026 at 20:27 UTC 4f11a440e95cea2fe0a0a3f24e142e4fb27df869
2 files changed +56 -1
ts/packages/cli/src/commands/tools/commands/tools.execute.cmd.ts
+7 -1
@@ -332,7 +332,13 @@ const getExecuteOutputEncoder = () => {
332 return executeOutputEncoder;
333 };
334
335 -const countOutputTokens = (json: string): number => getExecuteOutputEncoder().encode(json).length;
335 +// `Tiktoken.encode` defaults `disallowedSpecial` to "all", which makes it throw
336 +// on any tool response that happens to contain the literal text `<|endoftext|>`
337 +// or `<|endofprompt|>` (a README about tokenizers is enough). Here the encoder
338 +// is only a length gauge, so those literals are ordinary characters: passing
339 +// `allowedSpecial: 'all'` counts them instead of rejecting the payload.
340 +const countOutputTokens = (json: string): number =>
341 + getExecuteOutputEncoder().encode(json, 'all').length;
342
343 // A BPE token always covers at least one UTF-8 byte, so a payload of at most
344 // THRESHOLD bytes can never exceed THRESHOLD tokens. Checking the byte length
ts/packages/cli/test/src/commands/tools/tools.execute.cmd.test.ts
+49
@@ -1190,6 +1190,55 @@ describe('CLI: composio execute', () => {
1190 );
1191 }
1192 );
1193 + layer(
1194 + TestLive({
1195 + baseConfigProvider: largeOutputConfigProvider,
1196 + fixture: 'global-test-user-id',
1197 + stdin: { isTTY: true, data: '' },
1198 + toolsExecutor: {
1199 + respondWith: {
1200 + data: {
1201 + // Both of o200k's special tokens, in a payload past the inline
1202 + // threshold so the token count is actually computed. Reading a file
1203 + // that documents a tokenizer is enough to hit this in real use.
1204 + content: `<|endoftext|> <|endofprompt|> ${'token '.repeat(20_000)}`,
1205 + },
1206 + error: null,
1207 + successful: true,
1208 + logId: 'log_special_tokens',
1209 + },
1210 + },
1211 + })
1212 + )(
1213 + '[Given] a response containing tiktoken special-token literals [Then] it still reports the execution',
1214 + it => {
1215 + it.effect('counts the literals as text instead of failing the command', () =>
1216 + Effect.gen(function* () {
1217 + yield* cli(['execute', 'GMAIL_SEND_EMAIL', '-d', '{"recipient":"a"}']);
1218 + const lines = yield* MockConsole.getLines({ stripAnsi: true });
1219 + const output = parseLastJson(lines) as unknown as {
1220 + successful: boolean;
1221 + storedInFile: boolean;
1222 + tokenCount: number;
1223 + outputFilePath: string;
1224 + };
1225 +
1226 + expect(output.successful).toBe(true);
1227 + expect(output.storedInFile).toBe(true);
1228 + expect(output.tokenCount).toBeGreaterThan(10_000);
1229 +
1230 + const storedJson = fs.readFileSync(output.outputFilePath, 'utf8');
1231 + expect(storedJson).toContain('<|endoftext|>');
1232 + expect(storedJson).toContain('<|endofprompt|>');
1233 +
1234 + fs.rmSync(output.outputFilePath.slice(0, output.outputFilePath.lastIndexOf('/')), {
1235 + recursive: true,
1236 + force: true,
1237 + });
1238 + })
1239 + );
1240 + }
1241 + );
1242
1243 layer(
1244 TestLive({