Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":1,"fingerprint":"390d69b6c1dbd81c","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"which composio && composio search slack 2>&1 | head -40","exitCode":0},{"command":"composio link slack 2>&1 | head -30","exitCode":0},{"command":"composio execute \"SLACK_SEND_MESSAGE\" -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}' 2>&1 | head -60","exitCode":0}],"costUsd":0.002170395,"durationMs":9221} |
| 2 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":4,"fingerprint":"390d69b6c1dbd81c","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":9,"commands":[{"command":"command -v composio && composio --version 2>/dev/null; command -v slack; env | grep -i slack | sed 's/=.*/=<set>/'","exitCode":0},{"command":"composio whoami; composio search \"send a slack message to a channel\" --toolkits slack","exitCode":0},{"command":"composio link slack --no-browser --no-wait","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'","exitCode":0}],"costUsd":0.003294845,"durationMs":12372} |
| 3 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":2,"fingerprint":"390d69b6c1dbd81c","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio search \"send a slack message to a channel\"","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'","exitCode":1},{"command":"composio link slack --no-browser --no-wait","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'","exitCode":0}],"costUsd":0.00340621,"durationMs":12804} |
| 4 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":3,"fingerprint":"390d69b6c1dbd81c","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"command -v composio && composio --help 2>&1 | head -50","exitCode":0},{"command":"composio search \"send a slack message to a channel\" --human 2>&1 | head -40; echo ---; composio connections --help 2>&1 | head -30","exitCode":0},{"command":"composio connections list 2>&1 | head -60; echo ---; composio search \"slack send message channel\" 2>&1 | head -60","exitCode":0},{"command":"timeout 60 composio link slack 2>&1 | head -30","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}' 2>&1 | head -60","exitCode":0}],"costUsd":0.0031380949999999996,"durationMs":13333} |
| 5 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":5,"fingerprint":"390d69b6c1dbd81c","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":6,"commands":[{"command":"which composio slack 2>/dev/null; composio --version 2>/dev/null; echo \"exit=$?\"","exitCode":0},{"command":"composio search slack 2>&1 | head -40","exitCode":0},{"command":"composio link slack 2>&1 | head -30","exitCode":0},{"command":"cat ~/.composio/tool_definitions/SLACK_SEND_MESSAGE.json 2>/dev/null | head -60","exitCode":0},{"command":"composio execute \"SLACK_SEND_MESSAGE\" -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}' 2>&1 | head -60","exitCode":0}],"costUsd":0.0017231799999999997,"durationMs":10444} |
| 6 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":1,"fingerprint":"d16b9fdcb4283b95","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.003106775,"durationMs":12524} |
| 7 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":3,"fingerprint":"d16b9fdcb4283b95","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.002771805,"durationMs":12599} |
| 8 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":2,"fingerprint":"d16b9fdcb4283b95","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0028182100000000002,"durationMs":14035} |
| 9 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":4,"fingerprint":"d16b9fdcb4283b95","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0028031200000000005,"durationMs":13678} |
| 10 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":1,"fingerprint":"93cb421be0b2cb86","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"command -v composio && composio --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login --help 2>&1 | head -40","exitCode":0},{"command":"composio login 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login --poll 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0}],"costUsd":0.00195205,"durationMs":9877} |
| 11 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":5,"fingerprint":"d16b9fdcb4283b95","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":9,"commands":[{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0027455999999999995,"durationMs":12289} |
| 12 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":3,"fingerprint":"93cb421be0b2cb86","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login --poll 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0}],"costUsd":0.001429785,"durationMs":9232} |
| 13 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":4,"fingerprint":"93cb421be0b2cb86","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"command -v composio && composio --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login --help 2>&1 | head -30","exitCode":0},{"command":"timeout 120 composio login 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"timeout 620 composio login --poll 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0}],"costUsd":0.001956375,"durationMs":10544} |
| 14 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":2,"fingerprint":"93cb421be0b2cb86","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":false,"seen":"\"{\\\"provider\\\":\\\"deadline\\\"}\""},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":false,"seen":"\"[{\\\"id\\\":\\\"00000000-0000-4000-8000-000000050001\\\",\\\"code\\\":\\\"E2E001\\\",\\\"status\\\":\\\"pending\\\",\\\"reportedLinked\\\":false}]\""},{"name":"the login completed: the user API key was fetched for the linked session","ok":false,"seen":"null"},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":false,"seen":"\"{\\\"exitCode\\\":1,\\\"stdout\\\":\\\"\\\",\\\"stderr\\\":\\\"You are not logged in yet. Please run `composio login`.\\\\n\\\"}\""},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"deadline","modelTurns":0,"commands":[{"command":"composio whoami 2>&1; echo \"exit: $?\"","exitCode":0},{"command":"composio login 2>&1; echo \"exit: $?\"","exitCode":0}],"costUsd":0,"durationMs":361034} |
| 15 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":5,"fingerprint":"93cb421be0b2cb86","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":false,"seen":"\"{\\\"provider\\\":\\\"deadline\\\"}\""},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":false,"seen":"\"[{\\\"id\\\":\\\"00000000-0000-4000-8000-000000050001\\\",\\\"code\\\":\\\"E2E001\\\",\\\"status\\\":\\\"pending\\\",\\\"reportedLinked\\\":false}]\""},{"name":"the login completed: the user API key was fetched for the linked session","ok":false,"seen":"null"},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":false,"seen":"\"{\\\"exitCode\\\":1,\\\"stdout\\\":\\\"\\\",\\\"stderr\\\":\\\"You are not logged in yet. Please run `composio login`.\\\\n\\\"}\""},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"deadline","modelTurns":0,"commands":[{"command":"command -v composio && composio --help 2>&1 | head -50","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"timeout 120 composio login 2>&1; echo \"exit=$?\"","exitCode":0}],"costUsd":0,"durationMs":360805} |