Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 12:15 UTC
docs(cli): describe special-token literals as counted special tokens
encode(json, 'all') turns each literal into one special token rather than counting its characters as text, so say that in the comment and the test name.
jkomyno committed
Sep 14, 2026 at 14:04 UTC
98957265101cc97a45acad73f572a72508af0589
2 files changed
+3
-3
ts/packages/cli/src/commands/tools/commands/tools.execute.cmd.ts
+2
-2
@@ -335,8 +335,8 @@ const getExecuteOutputEncoder = () => {
335
// `Tiktoken.encode` defaults `disallowedSpecial` to "all", which makes it throw
336
// on any tool response that happens to contain the literal text `<|endoftext|>`
337
// or `<|endofprompt|>` (a README about tokenizers is enough). Here the encoder
338
-// is only a length gauge, so those literals are ordinary characters: passing
339
-// `allowedSpecial: 'all'` counts them instead of rejecting the payload.
338
+// is only a length gauge, so passing `allowedSpecial: 'all'` counts each literal
339
+// as the single special token it encodes to instead of rejecting the payload.
340
const countOutputTokens = (json: string): number =>
341
getExecuteOutputEncoder().encode(json, 'all').length;
342
ts/packages/cli/test/src/commands/tools/tools.execute.cmd.test.ts
+1
-1
@@ -1212,7 +1212,7 @@ describe('CLI: composio execute', () => {
1212
)(
1213
'[Given] a response containing tiktoken special-token literals [Then] it still reports the execution',
1214
it => {
1215
- it.effect('counts the literals as text instead of failing the command', () =>
1215
+ it.effect('counts the literals as special tokens instead of failing the command', () =>
1216
Effect.gen(function* () {
1217
yield* cli(['execute', 'GMAIL_SEND_EMAIL', '-d', '{"recipient":"a"}']);
1218
const lines = yield* MockConsole.getLines({ stripAnsi: true });