Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 07:15 UTC
| 1 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":2,"fingerprint":"4009c9ca4cbd9aa1","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"which slack composio slack-cli 2>/dev/null; env | grep -i slack | sed 's/=.*/=<set>/'; ls /home/journey 2>/dev/null | head -50","exitCode":0},{"command":"composio search \"slack send message to channel\" 2>&1 | head -40","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -60; echo ---; composio link slack 2>&1 | head -20","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}' 2>&1","exitCode":0}],"costUsd":0.00244207,"durationMs":9856} |
| 2 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":1,"fingerprint":"4009c9ca4cbd9aa1","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":9,"commands":[{"command":"which composio slack slack-cli 2>/dev/null; echo \"exit=$?\"","exitCode":0},{"command":"composio search \"send a slack message to a channel\" --toolkits slack; echo \"exit=$?\"; composio whoami; echo \"exit=$?\"","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema; echo \"exit=$?\"; composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'; echo \"exit=$?\"","exitCode":0},{"command":"composio link slack --no-browser --no-wait; echo \"exit=$?\"","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'; echo \"exit=$?\"","exitCode":0}],"costUsd":0.003546035,"durationMs":13244} |
| 3 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":3,"fingerprint":"4009c9ca4cbd9aa1","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":6,"commands":[{"command":"which composio && composio --help 2>&1 | head -50","exitCode":0},{"command":"composio search \"send a slack message to a channel\" 2>&1 | head -80; echo ---; composio connections --help 2>&1 | head -30","exitCode":0},{"command":"composio connections list 2>&1 | head -40","exitCode":0},{"command":"timeout 120 composio link slack 2>&1 | head -40","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}' 2>&1 | head -60","exitCode":0}],"costUsd":0.00160708,"durationMs":10116} |
| 4 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":4,"fingerprint":"4009c9ca4cbd9aa1","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami; composio search \"send a slack message to a channel\"","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --dry-run -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'","exitCode":1},{"command":"composio link slack --no-wait","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}'","exitCode":0}],"costUsd":0.00246845,"durationMs":11965} |
| 5 | {"journey":"consumer-slack-message","split":"train","arm":"skill","attempt":5,"fingerprint":"4009c9ca4cbd9aa1","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"deploy finished\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"which composio slack 2>/dev/null; composio --version 2>/dev/null; echo \"---\"; composio search slack 2>&1 | head -40","exitCode":0},{"command":"composio execute \"SLACK_SEND_MESSAGE\" -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}' 2>&1 | head -40","exitCode":0},{"command":"timeout 60 composio link slack 2>&1 | head -30","exitCode":0},{"command":"composio execute \"SLACK_SEND_MESSAGE\" -d '{\"channel\":\"#general\",\"text\":\"deploy finished\"}' 2>&1 | head -40","exitCode":0}],"costUsd":0.001329485,"durationMs":9502} |
| 6 | {"journey":"consumer-slack-message","split":"train","arm":"no-skill","attempt":1,"fingerprint":"24d806dcd9142dde","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":false,"seen":"\"[\\\"0 env | grep -i slack; which slack slack-cli 2>/dev/null; ls /home/journey/.config 2>/dev/null; ls -a /home/journey | head -50\\\",\\\"0 ls -la /home/journey/.composio; grep -il slack -r /home/journey/.composio /home/journey/.claude.json 2>/dev/null | head\\\"]\""},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"deploy finished\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":false,"seen":"null"},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":3,"commands":[{"command":"env | grep -i slack; which slack slack-cli 2>/dev/null; ls /home/journey/.config 2>/dev/null; ls -a /home/journey | head -50","exitCode":0},{"command":"ls -la /home/journey/.composio; grep -il slack -r /home/journey/.composio /home/journey/.claude.json 2>/dev/null | head","exitCode":0}],"costUsd":0.00109502,"durationMs":6288} |
| 7 | {"journey":"consumer-slack-message","split":"train","arm":"no-skill","attempt":2,"fingerprint":"24d806dcd9142dde","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":false,"seen":"\"[\\\"0 env | grep -i -E 'slack|webhook' | sed -E 's/=.*/=<set>/'; which slack slack-cli 2>/dev/null; ls /home/journey 2>/dev/null\\\"]\""},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"deploy finished\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":false,"seen":"null"},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":2,"commands":[{"command":"env | grep -i -E 'slack|webhook' | sed -E 's/=.*/=<set>/'; which slack slack-cli 2>/dev/null; ls /home/journey 2>/dev/null","exitCode":0}],"costUsd":0.0005951750000000001,"durationMs":5371} |
| 8 | {"journey":"consumer-slack-message","split":"train","arm":"no-skill","attempt":3,"fingerprint":"24d806dcd9142dde","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":false,"seen":"\"[\\\"0 env | grep -i slack | sed 's/=.*/=<set>/'; which slack slack-cli 2>/dev/null; ls /home/journey 2>/dev/null | head -50\\\"]\""},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"deploy finished\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":false,"seen":"null"},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":2,"commands":[{"command":"env | grep -i slack | sed 's/=.*/=<set>/'; which slack slack-cli 2>/dev/null; ls /home/journey 2>/dev/null | head -50","exitCode":0}],"costUsd":0.0005689250000000001,"durationMs":5067} |
| 9 | {"journey":"consumer-slack-message","split":"train","arm":"no-skill","attempt":4,"fingerprint":"24d806dcd9142dde","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":false,"seen":"\"[\\\"0 which slack slack-cli slackcat 2>/dev/null; env | grep -i slack | sed 's/=.*/=<set>/'; ls /home/journey/.claude 2>/dev/null\\\",\\\"0 grep -ril \\\\\\\"slack\\\\\\\" /home/journey/.claude.json /home/journey/.claude/settings*.json /home/journey/.config 2>/dev/null | head; echo \\\\\\\"exit=$?\\\\\\\"\\\"]\""},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"deploy finished\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":false,"seen":"null"},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":3,"commands":[{"command":"which slack slack-cli slackcat 2>/dev/null; env | grep -i slack | sed 's/=.*/=<set>/'; ls /home/journey/.claude 2>/dev/null","exitCode":0},{"command":"grep -ril \"slack\" /home/journey/.claude.json /home/journey/.claude/settings*.json /home/journey/.config 2>/dev/null | head; echo \"exit=$?\"","exitCode":0}],"costUsd":0.000723035,"durationMs":6116} |
| 10 | {"journey":"consumer-slack-message","split":"train","arm":"no-skill","attempt":5,"fingerprint":"24d806dcd9142dde","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the last `composio execute` exited 0","ok":false,"seen":"\"[\\\"0 env | grep -i -E 'slack|webhook' | sed -E 's/=.*/=<set>/'; command -v slack slack-cli slackcli 2>/dev/null; ls /home/journey 2>/dev/null | head -50\\\"]\""},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"deploy finished\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the consumer project","ok":false,"seen":"null"},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":2,"commands":[{"command":"env | grep -i -E 'slack|webhook' | sed -E 's/=.*/=<set>/'; command -v slack slack-cli slackcli 2>/dev/null; ls /home/journey 2>/dev/null | head -50","exitCode":0}],"costUsd":0.000565425,"durationMs":5346} |
| 11 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"skill","attempt":1,"fingerprint":"3ba31957d23e2722","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"composio whoami; composio dev auth-configs list","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.00253856,"durationMs":11038} |
| 12 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"skill","attempt":2,"fingerprint":"3ba31957d23e2722","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"composio dev auth-configs list","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0025137799999999997,"durationMs":10424} |
| 13 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"skill","attempt":3,"fingerprint":"3ba31957d23e2722","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"composio dev auth-configs list","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0018239,"durationMs":9179} |
| 14 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"skill","attempt":4,"fingerprint":"3ba31957d23e2722","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"composio dev auth-configs list","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.00188405,"durationMs":9208} |
| 15 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"skill","attempt":5,"fingerprint":"3ba31957d23e2722","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"composio dev auth-configs list","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0016008449999999998,"durationMs":9673} |
| 16 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"no-skill","attempt":1,"fingerprint":"e2c0581ea7dade0a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":8,"commands":[{"command":"ls -la && (which composio; composio --version) 2>&1 | head -5; grep -ril composio --exclude-dir=node_modules --exclude-dir=.git . 2>/dev/null | head -20","exitCode":0},{"command":"ls -la .composio && cat .composio/project.json && echo --- && sed -E 's/=.*/=<redacted>/' .env.local && echo --- && composio --help 2>&1 | head -60","exitCode":0},{"command":"composio dev --help 2>&1 | head -60; echo ---; composio whoami 2>&1 | head -20","exitCode":0},{"command":"composio dev auth-configs --help 2>&1 | head -40; echo ---; composio dev auth-configs list 2>&1 | head -80","exitCode":0},{"command":"composio dev playground-execute --help 2>&1 | head -50; echo ---; composio execute --help 2>&1 | head -40; echo ---; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --dry-run -d '{\"channel\":\"general\",\"text\":\"hello from dev\"}' 2>&1 | head -60","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{\"channel\":\"general\",\"text\":\"hello from dev\"}' 2>&1 | head -80","exitCode":0}],"costUsd":0.0032672100000000004,"durationMs":16870} |
| 17 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"no-skill","attempt":2,"fingerprint":"e2c0581ea7dade0a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":8,"commands":[{"command":"ls -la && (which composio; composio --version) 2>&1; env | grep -i composio | sed 's/=.*/=<set>/'","exitCode":0},{"command":"composio --help 2>&1 | head -60; echo ---; cat /work/app/.composio/project.json 2>&1","exitCode":0},{"command":"composio dev --help 2>&1 | head -60; echo ---; composio whoami 2>&1 | head -20","exitCode":0},{"command":"composio dev auth-configs --help 2>&1 | head -40; echo ---; composio dev auth-configs list 2>&1 | head -80","exitCode":0},{"command":"composio dev playground-execute --help 2>&1 | head -50; echo ---; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1; echo \"--- EXECUTE ---\"; composio dev playground-execute SLACK_SEND_MESSAGE -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1","exitCode":0}],"costUsd":0.0024921,"durationMs":16965} |
| 18 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"no-skill","attempt":3,"fingerprint":"e2c0581ea7dade0a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":14,"commands":[{"command":"ls -la && (command -v composio || echo \"composio CLI not found\") && grep -ril composio --include=*.{py,ts,js,json,toml,yaml,yml,env,md} . 2>/dev/null | head -n 20","exitCode":0},{"command":"cat .composio/project.json && echo --- && sed -E 's/=(.{4}).*/=\\1…(redacted)/' .env.local && echo --- && composio --help 2>&1 | head -n 60","exitCode":0},{"command":"composio dev --help 2>&1 | head -n 60; echo ---; composio whoami 2>&1 | head -n 20","exitCode":0},{"command":"composio dev auth-configs --help 2>&1 | head -n 40; echo ---; composio dev auth-configs list 2>&1 | head -n 80","exitCode":0},{"command":"composio execute --help 2>&1 | head -n 50; echo ---; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -n 80","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --user-id pg-test-00000000-0000-4000-8000-00000000a001 --account ac_e2e0001 --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -n 60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --account ac_e2e0001 --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -n 60","exitCode":0},{"command":"composio dev connected-accounts list 2>&1 | head -n 80; echo ---; composio connections --help 2>&1 | head -n 25; echo ---; composio connections list 2>&1 | head -n 60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --account ca_e2e0001 --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -n 60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -n 60; echo \"exit=$?\"","exitCode":0},{"command":"composio dev playground-execute --help 2>&1 | head -n 60","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -n 40","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -n 60","exitCode":0}],"costUsd":0.005374005,"durationMs":32422} |
| 19 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"no-skill","attempt":4,"fingerprint":"e2c0581ea7dade0a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[{"name":"no root `composio execute` tried to run the tool","ok":false,"seen":"\"[\\\"composio execute SLACK_SEND_MESSAGE -d '{\\\\\\\"channel\\\\\\\":\\\\\\\"#general\\\\\\\",\\\\\\\"text\\\\\\\":\\\\\\\"hello from dev\\\\\\\"}' 2>&1 | head -80\\\",\\\"composio execute SLACK_SEND_MESSAGE --user-id pg-test-00000000-0000-4000-8000-00000000a001 -d '{\\\\\\\"channel\\\\\\\":\\\\\\\"#general\\\\\\\",\\\\\\\"text\\\\\\\":\\\\\\\"hello from dev\\\\\\\"}' 2>&1 | head -80\\\",\\\"composio execute SLACK_SEND_MESSAGE --account slack_e2e_0001 -d '{\\\\\\\"channel\\\\\\\":\\\\\\\"#general\\\\\\\",\\\\\\\"text\\\\\\\":\\\\\\\"hello from dev\\\\\\\"}' 2>&1 | head -80\\\"]\""},{"name":"nothing was executed or linked on the consumer project, completed or not","ok":false,"seen":"null"}],"stop":"finished","modelTurns":12,"commands":[{"command":"ls -la && (which composio && composio --version) 2>&1; env | grep -i composio | sed 's/=.*/=<set>/'","exitCode":0},{"command":"composio --help 2>&1 | head -60; echo ---; composio auth-configs --help 2>&1 | head -40; echo ---; composio whoami 2>&1 | head -20","exitCode":0},{"command":"composio dev --help 2>&1 | head -50; echo ---; composio dev accounts --help 2>&1 | head -30","exitCode":0},{"command":"composio dev auth-configs list 2>&1 | head -80; echo ---; cat .composio/* 2>/dev/null | head -30; ls -la .composio","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE -d '{\"channel\":\"#general\",\"text\":\"hello from dev\"}' 2>&1 | head -80","exitCode":0},{"command":"composio dev connected-accounts list 2>&1 | head -60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --user-id pg-test-00000000-0000-4000-8000-00000000a001 -d '{\"channel\":\"#general\",\"text\":\"hello from dev\"}' 2>&1 | head -80","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --account slack_e2e_0001 -d '{\"channel\":\"#general\",\"text\":\"hello from dev\"}' 2>&1 | head -80","exitCode":0},{"command":"composio dev playground-execute --help 2>&1 | head -60","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --user-id pg-test-00000000-0000-4000-8000-00000000a001 -d '{\"channel\":\"#general\",\"text\":\"hello from dev\"}' 2>&1 | head -80","exitCode":0}],"costUsd":0.00426277,"durationMs":24479} |
| 20 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":1,"fingerprint":"5aa12b1580290687","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0028595,"durationMs":12205} |
| 21 | {"journey":"developer-auth-configs-and-execute","split":"holdout","arm":"no-skill","attempt":5,"fingerprint":"e2c0581ea7dade0a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"`composio dev auth-configs list` ran and exited 0","ok":true},{"name":"the auth configs were read from the developer project","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"it ran through `composio dev playground-execute`","ok":true},{"name":"no account was connected outside the developer project","ok":true}],"detours":[],"stop":"finished","modelTurns":9,"commands":[{"command":"ls -la && (which composio; composio --version) 2>&1 | head -20; env | grep -i composio | sed 's/=.*/=<set>/'","exitCode":0},{"command":"composio --help 2>&1 | head -60; echo ---; composio auth-configs --help 2>&1 | head -40; echo ---; composio execute --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1 | head -30; echo ---; composio dev --help 2>&1 | sed -n '1,60p'","exitCode":0},{"command":"composio dev auth-configs --help 2>&1 | sed -n '1,40p'; echo ---; composio dev auth-configs list 2>&1 | head -80","exitCode":0},{"command":"composio dev playground-execute --help 2>&1 | sed -n '1,60p'","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --dry-run -d '{ channel: \"general\", markdown_text: \"hello from dev\" }' 2>&1 | head -60","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --dry-run -d '{ channel: \"general\", text: \"hello from dev\" }' 2>&1 | head -60","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"general\", text: \"hello from dev\" }' 2>&1 | head -80","exitCode":0}],"costUsd":0.0032457199999999997,"durationMs":16308} |
| 22 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":2,"fingerprint":"5aa12b1580290687","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami; echo \"---\"; composio dev --help 2>&1 | head -60","exitCode":0},{"command":"composio dev playground-execute --help 2>&1 | head -80","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema 2>&1","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"general\", text: \"hello from dev\" }' 2>&1","exitCode":1},{"command":"composio dev connected-accounts link slack --no-wait 2>&1","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"general\", text: \"hello from dev\" }' 2>&1","exitCode":0}],"costUsd":0.00254849,"durationMs":13115} |
| 23 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":3,"fingerprint":"5aa12b1580290687","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.0024563750000000002,"durationMs":12764} |
| 24 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":4,"fingerprint":"5aa12b1580290687","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":9,"commands":[{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.002490045,"durationMs":12091} |
| 25 | {"journey":"developer-execute-unconnected","split":"train","arm":"skill","attempt":5,"fingerprint":"5aa12b1580290687","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":10,"commands":[{"command":"composio whoami","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":1},{"command":"composio dev connected-accounts link slack --no-browser --no-wait","exitCode":0},{"command":"composio dev playground-execute SLACK_SEND_MESSAGE -d '{ channel: \"#general\", text: \"hello from dev\" }'","exitCode":0}],"costUsd":0.002653685,"durationMs":12219} |
| 26 | {"journey":"developer-execute-unconnected","split":"train","arm":"no-skill","attempt":1,"fingerprint":"b418fae0feb80061","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"hello from dev\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":false,"seen":"null"},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":false,"seen":"\"[\\\"0 ls -la && (ls -la .composio 2>/dev/null; cat .env 2>/dev/null | sed -E 's/=.*/=<redacted>/'; grep -ril composio --include=*.py --include=*.ts --include=*.js --include=*.json --include=*.md --include=*.yaml --include=*.toml . 2>/dev/null | grep -v node_modules | head -30)\\\",\\\"0 cat .composio/project.json; echo; cat .composio/.gitignore; echo; sed -E 's/=.*/=<redacted>/' .env.local; echo; which composio; composio --version 2>&1 | head -3\\\",\\\"0 composio --help 2>&1 | head -60; echo ---; compo"}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"ls -la && (ls -la .composio 2>/dev/null; cat .env 2>/dev/null | sed -E 's/=.*/=<redacted>/'; grep -ril composio --include=*.py --include=*.ts --include=*.js --include=*.json --include=*.md --include=*.yaml --include=*.toml . 2>/dev/null | grep -v node_modules | head -30)","exitCode":0},{"command":"cat .composio/project.json; echo; cat .composio/.gitignore; echo; sed -E 's/=.*/=<redacted>/' .env.local; echo; which composio; composio --version 2>&1 | head -3","exitCode":0},{"command":"composio --help 2>&1 | head -60; echo ---; composio execute --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1 | head -20; echo ---; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80; echo ---; composio execute SLACK_SEND_MESSAGE --dry-run -d '{ channel: \"general\", markdown_text: \"hello from dev\" }' 2>&1 | head -60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --dry-run -d '{ channel: \"general\", text: \"hello from dev\" }' 2>&1 | sed 's/\\x1b\\[[0-9;]*m//g' | head -60; echo ---; composio connections --help 2>&1 | sed 's/\\x1b\\[[0-9;]*m//g' | head -30","exitCode":0},{"command":"composio connections list 2>&1 | sed 's/\\x1b\\[[0-9;]*m//g' | head -60","exitCode":0}],"costUsd":0.003115955,"durationMs":20205} |
| 27 | {"journey":"developer-execute-unconnected","split":"train","arm":"no-skill","attempt":2,"fingerprint":"b418fae0feb80061","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"hello from dev\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":true},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":true}],"detours":[],"stop":"finished","modelTurns":16,"commands":[{"command":"ls -la && cat .env 2>/dev/null | sed 's/=.*/=<redacted>/' ; grep -ril composio --include=*.py --include=*.ts --include=*.js --include=*.json --include=*.md --include=*.toml --include=*.yaml . 2>/dev/null | grep -v node_modules | head -30","exitCode":0},{"command":"cat .composio/project.json; echo; ls -la .composio; sed 's/=.*/=<redacted>/' .env.local; echo; which composio; composio --version 2>&1 | head -3","exitCode":0},{"command":"grep -o 'COMPOSIO_TEST_USER_ID=.*' .env.local | sed 's/=\\(.\\{0,8\\}\\).*/=\\1…/'; composio --help 2>&1 | head -60","exitCode":0},{"command":"composio execute --help 2>&1 | head -80; echo ---; composio whoami 2>&1 | head -20","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80; echo ---; composio execute SLACK_SEND_MESSAGE --dry-run -d '{\"channel\":\"general\",\"markdown_text\":\"hello from dev\"}' 2>&1 | head -60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --dry-run -d '{\"channel\":\"general\",\"text\":\"hello from dev\"}' 2>&1 | head -60; echo ---; composio connections --help 2>&1 | head -40; echo ---; composio config --help 2>&1 | head -30","exitCode":0},{"command":"composio connections list 2>&1 | head -60; echo ---; env | grep -i composio | sed 's/=.*/=<set>/'","exitCode":0},{"command":"composio dev --help 2>&1 | head -40; echo ---; composio whoami --help 2>&1 | head -20; echo ---; composio execute SLACK_SEND_MESSAGE --help 2>&1 | grep -i user","exitCode":0},{"command":"composio dev playground-execute --help 2>&1 | head -60","exitCode":0},{"command":"TU=$(grep -o '\"test_user_id\":\"[^\"]*\"' .composio/project.json | cut -d'\"' -f4); composio dev playground-execute SLACK_SEND_MESSAGE --user-id \"$TU\" --dry-run -d '{\"channel\":\"general\",\"text\":\"hello from dev\"}' 2>&1 | head -40","exitCode":0},{"command":"TU=$(grep -o '\"test_user_id\":\"[^\"]*\"' .composio/project.json | cut -d'\"' -f4); composio dev playground-execute SLACK_SEND_MESSAGE --user-id \"$TU\" -d '{\"channel\":\"general\",\"text\":\"hello from dev\"}' 2>&1 | head -60","exitCode":0},{"command":"composio dev connected-accounts link --help 2>&1 | head -40","exitCode":0},{"command":"TU=$(grep -o '\"test_user_id\":\"[^\"]*\"' .composio/project.json | cut -d'\"' -f4); timeout 60 composio dev connected-accounts link slack --user-id \"$TU\" --no-browser --no-wait 2>&1 | head -30","exitCode":0},{"command":"TU=pg-test-00000000-0000-4000-8000-00000000a001; composio dev connected-accounts link slack --user-id \"$TU\" --list 2>&1 | head -30; echo ---; composio dev playground-execute SLACK_SEND_MESSAGE --user-id \"$TU\" -d '{\"channel\":\"general\",\"text\":\"hello from dev\"}' 2>&1 | head -60","exitCode":0}],"costUsd":0.00602479,"durationMs":39628} |
| 28 | {"journey":"developer-execute-unconnected","split":"train","arm":"no-skill","attempt":3,"fingerprint":"b418fae0feb80061","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"hello from dev\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":false,"seen":"null"},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":false,"seen":"\"[\\\"1 ls -la && (ls -la .composio* composio* 2>/dev/null; cat .env 2>/dev/null | sed 's/=.*/=<redacted>/'; which composio; cat composio.yaml 2>/dev/null)\\\",\\\"0 cat .composio/project.json; echo; sed 's/=.*/=<redacted>/' .env.local; echo; composio --help 2>&1 | head -60\\\",\\\"0 composio execute --help 2>&1 | head -60; echo ---; composio whoami 2>&1 | head -20\\\",\\\"0 composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80; echo ---; composio execute SLACK_SEND_MESSAGE --dry-run -d '{ \\\\\\\"chan"}],"detours":[],"stop":"finished","modelTurns":9,"commands":[{"command":"ls -la && (ls -la .composio* composio* 2>/dev/null; cat .env 2>/dev/null | sed 's/=.*/=<redacted>/'; which composio; cat composio.yaml 2>/dev/null)","exitCode":1},{"command":"cat .composio/project.json; echo; sed 's/=.*/=<redacted>/' .env.local; echo; composio --help 2>&1 | head -60","exitCode":0},{"command":"composio execute --help 2>&1 | head -60; echo ---; composio whoami 2>&1 | head -20","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80; echo ---; composio execute SLACK_SEND_MESSAGE --dry-run -d '{ \"channel\": \"general\", \"markdown_text\": \"hello from dev\" }' 2>&1 | head -60","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -40; echo ---; composio connections --help 2>&1 | head -30; echo ---; composio connections list 2>&1 | head -40","exitCode":0},{"command":"grep -o 'COMPOSIO_TEST_USER_ID=.*' .env.local; composio config --help 2>&1 | head -30; echo ---; composio config list 2>&1 | head -40","exitCode":0},{"command":"composio run --help 2>&1 | head -50","exitCode":0}],"costUsd":0.003273875,"durationMs":21618} |
| 29 | {"journey":"developer-execute-unconnected","split":"train","arm":"no-skill","attempt":4,"fingerprint":"b418fae0feb80061","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"hello from dev\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":false,"seen":"null"},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":false,"seen":"\"[\\\"0 ls -la && (grep -ril composio --include=* . 2>/dev/null | grep -v node_modules | head -50)\\\",\\\"0 cat .composio/project.json; echo ---; ls -la .composio; echo ---; sed -E 's/(KEY|SECRET|TOKEN)=.*/\\\\\\\\1=<redacted>/I' .env.local; echo ---; which composio; composio --version 2>&1 | head -3\\\",\\\"0 composio --help 2>&1 | head -60; echo ---; composio execute --help 2>&1 | head -60\\\",\\\"0 composio whoami 2>&1 | head -20; echo ---; composio connections --help 2>&1 | sed -n '1,30p'; echo ---; composio"}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"ls -la && (grep -ril composio --include=* . 2>/dev/null | grep -v node_modules | head -50)","exitCode":0},{"command":"cat .composio/project.json; echo ---; ls -la .composio; echo ---; sed -E 's/(KEY|SECRET|TOKEN)=.*/\\1=<redacted>/I' .env.local; echo ---; which composio; composio --version 2>&1 | head -3","exitCode":0},{"command":"composio --help 2>&1 | head -60; echo ---; composio execute --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1 | head -20; echo ---; composio connections --help 2>&1 | sed -n '1,30p'; echo ---; composio connections list 2>&1 | head -40; echo ---; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --dry-run -d '{ \"channel\": \"general\", \"text\": \"hello from dev\" }' 2>&1 | head -60","exitCode":0},{"command":"composio config --help 2>&1 | sed -n '1,25p'; echo ---; composio config list 2>&1 | head -30; echo ---; env | grep -i composio | sed -E 's/(KEY|SECRET|TOKEN)=.*/\\1=<redacted>/I'","exitCode":0}],"costUsd":0.002831385,"durationMs":25723} |
| 30 | {"journey":"developer-execute-unconnected","split":"train","arm":"no-skill","attempt":5,"fingerprint":"b418fae0feb80061","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":false,"seen":"\"[]\""},{"name":"it ran on the developer project, as pg-test-00000000-0000-4000-8000-00000000a001","ok":false,"seen":"\"[]\""},{"name":"it went to #general","ok":false,"seen":"\"[]\""},{"name":"its text was exactly \"hello from dev\"","ok":false,"seen":"\"[]\""},{"name":"no other tool that changes something ran","ok":true},{"name":"the account was linked on the developer project for the test user","ok":false,"seen":"null"},{"name":"no account was connected outside the developer project","ok":true},{"name":"the last `composio dev playground-execute` exited 0","ok":false,"seen":"\"[\\\"0 ls -la && (ls composio* .env* 2>/dev/null; cat .env 2>/dev/null | sed 's/=.*/=<redacted>/'; grep -ril composio --exclude-dir=node_modules --exclude-dir=.git . | head -50)\\\",\\\"0 ls -la .composio && cat .composio/project.json; echo; sed 's/=.*/=<redacted>/' .env.local; echo; which composio; composio --version 2>&1 | head -3\\\",\\\"0 composio --help 2>&1 | head -60; echo ---; composio execute --help 2>&1 | head -60\\\",\\\"0 composio whoami 2>&1 | head -20; echo ---; composio connections --help 2>&1"}],"detours":[],"stop":"finished","modelTurns":7,"commands":[{"command":"ls -la && (ls composio* .env* 2>/dev/null; cat .env 2>/dev/null | sed 's/=.*/=<redacted>/'; grep -ril composio --exclude-dir=node_modules --exclude-dir=.git . | head -50)","exitCode":0},{"command":"ls -la .composio && cat .composio/project.json; echo; sed 's/=.*/=<redacted>/' .env.local; echo; which composio; composio --version 2>&1 | head -3","exitCode":0},{"command":"composio --help 2>&1 | head -60; echo ---; composio execute --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1 | head -20; echo ---; composio connections --help 2>&1 | head -30; echo ---; composio execute SLACK_SEND_MESSAGE --dry-run -d '{ channel: \"general\", markdown_text: \"hello from dev\" }' 2>&1 | head -60","exitCode":0},{"command":"composio connections list 2>&1 | head -60; echo ---; composio execute SLACK_SEND_MESSAGE --dry-run -d '{ channel: \"general\", text: \"hello from dev\" }' 2>&1 | head -40","exitCode":0},{"command":"grep COMPOSIO_TEST_USER_ID /work/app/.env.local; echo ---; composio config --help 2>&1 | head -25; echo ---; composio config list 2>&1 | head -30","exitCode":0}],"costUsd":0.00283194,"durationMs":25201} |
| 31 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":1,"fingerprint":"8267c289733cffc4","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":false,"seen":"\"{\\\"provider\\\":\\\"deadline\\\"}\""},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":false,"seen":"\"[{\\\"id\\\":\\\"00000000-0000-4000-8000-000000050001\\\",\\\"code\\\":\\\"E2E001\\\",\\\"status\\\":\\\"pending\\\",\\\"reportedLinked\\\":false}]\""},{"name":"the login completed: the user API key was fetched for the linked session","ok":false,"seen":"null"},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":false,"seen":"\"{\\\"exitCode\\\":1,\\\"stdout\\\":\\\"\\\",\\\"stderr\\\":\\\"You are not logged in yet. Please run `composio login`.\\\\n\\\"}\""},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"deadline","modelTurns":0,"commands":[{"command":"which composio && composio --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login --help 2>&1","exitCode":0},{"command":"composio login --no-wait --no-skill-install 2>&1; echo \"exit=$?\"","exitCode":0}],"costUsd":0,"durationMs":362637} |
| 32 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":2,"fingerprint":"8267c289733cffc4","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":false,"seen":"\"{\\\"provider\\\":\\\"deadline\\\"}\""},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":false,"seen":"\"[{\\\"id\\\":\\\"00000000-0000-4000-8000-000000050001\\\",\\\"code\\\":\\\"E2E001\\\",\\\"status\\\":\\\"pending\\\",\\\"reportedLinked\\\":false}]\""},{"name":"the login completed: the user API key was fetched for the linked session","ok":false,"seen":"null"},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":false,"seen":"\"{\\\"exitCode\\\":1,\\\"stdout\\\":\\\"\\\",\\\"stderr\\\":\\\"You are not logged in yet. Please run `composio login`.\\\\n\\\"}\""},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"deadline","modelTurns":0,"commands":[{"command":"command -v composio && composio --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login --help 2>&1 | head -30","exitCode":0},{"command":"timeout 120 composio login 2>&1; echo \"exit=$?\"","exitCode":0}],"costUsd":0,"durationMs":362009} |
| 33 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":4,"fingerprint":"8267c289733cffc4","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"command -v composio && composio --version; composio whoami 2>&1; echo \"exit: $?\"","exitCode":0},{"command":"timeout 110 composio login 2>&1; echo \"exit: $?\"","exitCode":0},{"command":"timeout 600 composio login --poll 2>&1; echo \"exit: $?\"","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit: $?\"","exitCode":0}],"costUsd":0.00241503,"durationMs":8872} |
| 34 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":3,"fingerprint":"8267c289733cffc4","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":6,"commands":[{"command":"command -v composio && composio --help 2>&1 | head -60","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"timeout 60 composio login 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio login --poll 2>&1; echo \"exit=$?\"","exitCode":0},{"command":"composio whoami 2>&1; echo \"exit=$?\"","exitCode":0}],"costUsd":0.0027552749999999997,"durationMs":9809} |
| 35 | {"journey":"first-run-login","split":"train","arm":"no-skill","attempt":1,"fingerprint":"34bc8c4021a34a4f","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"which composio && composio --version; composio whoami 2>&1 | head -50","exitCode":0},{"command":"composio login 2>&1 | head -60","exitCode":0},{"command":"composio login --poll 2>&1 | tail -40","exitCode":0},{"command":"composio whoami 2>&1 | head -20","exitCode":0}],"costUsd":0.0013505300000000002,"durationMs":7451} |
| 36 | {"journey":"first-run-login","split":"train","arm":"no-skill","attempt":2,"fingerprint":"34bc8c4021a34a4f","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"which composio && composio --version; composio whoami 2>&1 | head -50","exitCode":0},{"command":"composio login 2>&1 | head -60","exitCode":0},{"command":"composio login --poll 2>&1 | tail -30; echo \"---\"; composio whoami 2>&1 | tail -10","exitCode":0}],"costUsd":0.000943525,"durationMs":6906} |
| 37 | {"journey":"first-run-login","split":"train","arm":"skill","attempt":5,"fingerprint":"8267c289733cffc4","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":false,"seen":"\"{\\\"provider\\\":\\\"deadline\\\"}\""},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":false,"seen":"\"[{\\\"id\\\":\\\"00000000-0000-4000-8000-000000050001\\\",\\\"code\\\":\\\"E2E001\\\",\\\"status\\\":\\\"pending\\\",\\\"reportedLinked\\\":false}]\""},{"name":"the login completed: the user API key was fetched for the linked session","ok":false,"seen":"null"},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":false,"seen":"\"{\\\"exitCode\\\":1,\\\"stdout\\\":\\\"\\\",\\\"stderr\\\":\\\"You are not logged in yet. Please run `composio login`.\\\\n\\\"}\""},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"deadline","modelTurns":0,"commands":[{"command":"command -v composio && composio --version; composio whoami 2>&1 | head -50","exitCode":0},{"command":"composio login 2>&1 | head -60","exitCode":0}],"costUsd":0,"durationMs":360805} |
| 38 | {"journey":"first-run-login","split":"train","arm":"no-skill","attempt":3,"fingerprint":"34bc8c4021a34a4f","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":false,"seen":"\"{\\\"provider\\\":\\\"deadline\\\"}\""},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":false,"seen":"\"[{\\\"id\\\":\\\"00000000-0000-4000-8000-000000050001\\\",\\\"code\\\":\\\"E2E001\\\",\\\"status\\\":\\\"pending\\\",\\\"reportedLinked\\\":false}]\""},{"name":"the login completed: the user API key was fetched for the linked session","ok":false,"seen":"null"},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":false,"seen":"\"{\\\"exitCode\\\":1,\\\"stdout\\\":\\\"\\\",\\\"stderr\\\":\\\"You are not logged in yet. Please run `composio login`.\\\\n\\\"}\""},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"deadline","modelTurns":0,"commands":[{"command":"which composio; composio --version 2>&1 | head -5; composio whoami 2>&1 | head -20","exitCode":0},{"command":"composio login 2>&1 | head -40","exitCode":0}],"costUsd":0,"durationMs":360504} |
| 39 | {"journey":"first-run-login","split":"train","arm":"no-skill","attempt":5,"fingerprint":"34bc8c4021a34a4f","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":true},{"name":"the login completed: the user API key was fetched for the linked session","ok":true},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":true},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"command -v composio && composio --version; composio whoami 2>&1 | head -n 20","exitCode":0},{"command":"composio login 2>&1 | head -n 40","exitCode":0},{"command":"composio login --poll 2>&1 | tail -n 30","exitCode":0},{"command":"composio whoami 2>&1 | head -n 20","exitCode":0}],"costUsd":0.001078215,"durationMs":7843} |
| 40 | {"journey":"run-inline-script","split":"holdout","arm":"skill","attempt":1,"fingerprint":"4b9dfd1cf61eb76a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"composio run '\n const res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"#general\", text: \"build passed\" });\n console.log(JSON.stringify(res, null, 2));\n console.log(\"timestamp:\", res.data?.ts ?? res.data?.message?.ts ?? \"not found in response\");\n'","exitCode":0}],"costUsd":0.001465165,"durationMs":6957} |
| 41 | {"journey":"run-inline-script","split":"holdout","arm":"skill","attempt":2,"fingerprint":"4b9dfd1cf61eb76a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"composio whoami; composio execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio run '\n const res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\n console.log(JSON.stringify(res.data ?? res, null, 2));\n const ts = res.data?.ts ?? res.data?.message?.ts ?? res.data?.data?.ts;\n console.log(\"timestamp:\", ts);\n'","exitCode":0}],"costUsd":0.001232845,"durationMs":6975} |
| 42 | {"journey":"run-inline-script","split":"holdout","arm":"skill","attempt":3,"fingerprint":"4b9dfd1cf61eb76a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":6,"commands":[{"command":"composio whoami && composio search \"send a message to a slack channel\" --toolkits slack","exitCode":0},{"command":"composio execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio run '\n const res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\n const ts = res.data?.ts ?? res.data?.message?.ts ?? res.data?.timestamp;\n console.log(ts ?? JSON.stringify(res));\n'","exitCode":0}],"costUsd":0.0015433550000000002,"durationMs":8419} |
| 43 | {"journey":"run-inline-script","split":"holdout","arm":"skill","attempt":4,"fingerprint":"4b9dfd1cf61eb76a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"composio execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio run '\n const res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"#general\", text: \"build passed\" });\n console.log(JSON.stringify(res.data ?? res, null, 2));\n'","exitCode":0}],"costUsd":0.00133452,"durationMs":7625} |
| 44 | {"journey":"run-inline-script","split":"holdout","arm":"skill","attempt":5,"fingerprint":"4b9dfd1cf61eb76a","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":5,"commands":[{"command":"composio whoami; composio execute SLACK_SEND_MESSAGE --get-schema","exitCode":0},{"command":"composio run '\n const res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\n const ts = res.data?.ts ?? res.data?.message?.ts ?? res.data?.timestamp;\n console.log(\"timestamp:\", ts ?? JSON.stringify(res));\n'","exitCode":0}],"costUsd":0.0009455900000000001,"durationMs":7187} |
| 45 | {"journey":"run-inline-script","split":"holdout","arm":"no-skill","attempt":1,"fingerprint":"fcad1d2487ef263e","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"composio --version && composio run --help","exitCode":0},{"command":"composio search \"send slack message\" 2>&1 | head -30; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -60","exitCode":0},{"command":"composio run '\nconst res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\nconsole.log(JSON.stringify(res, null, 2));\nconst ts = res?.data?.ts ?? res?.data?.message?.ts ?? res?.ts;\nconsole.log(\"timestamp:\", ts);\n' 2>&1","exitCode":0}],"costUsd":0.001439035,"durationMs":7391} |
| 46 | {"journey":"run-inline-script","split":"holdout","arm":"no-skill","attempt":2,"fingerprint":"fcad1d2487ef263e","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"which composio && composio --version; composio run --help 2>&1 | head -60","exitCode":0},{"command":"composio search \"send slack message\" 2>&1 | head -40; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80","exitCode":0},{"command":"composio run '\nconst res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\nconst ts = res?.data?.ts ?? res?.data?.message?.ts ?? res?.ts ?? res?.message?.ts;\nconsole.log(ts ?? JSON.stringify(res));\n' 2>&1","exitCode":0}],"costUsd":0.001139695,"durationMs":9097} |
| 47 | {"journey":"run-inline-script","split":"holdout","arm":"no-skill","attempt":3,"fingerprint":"fcad1d2487ef263e","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"composio run --help 2>&1 | head -60","exitCode":0},{"command":"composio search \"send slack message\" 2>&1 | head -30; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80","exitCode":0},{"command":"composio run '\nconst res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\nconst ts = res?.data?.ts ?? res?.data?.message?.ts ?? res?.ts ?? res?.message?.ts;\nconsole.log(ts ?? JSON.stringify(res));\n' 2>&1","exitCode":0}],"costUsd":0.0012266150000000001,"durationMs":8349} |
| 48 | {"journey":"run-inline-script","split":"holdout","arm":"no-skill","attempt":4,"fingerprint":"fcad1d2487ef263e","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"composio run --help 2>&1 | head -60","exitCode":0},{"command":"composio search \"send slack message\" 2>&1 | head -40; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -80","exitCode":0},{"command":"composio run '\nconst res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\nconst ts = res?.data?.ts ?? res?.data?.message?.ts ?? res?.ts ?? res?.message?.ts;\nconsole.log(JSON.stringify(res, null, 2));\nconsole.log(\"timestamp:\", ts);\n' 2>&1","exitCode":0}],"costUsd":0.00114928,"durationMs":7009} |
| 49 | {"journey":"run-inline-script","split":"holdout","arm":"no-skill","attempt":5,"fingerprint":"fcad1d2487ef263e","outcome":"passed","checks":[{"name":"the model finished by itself, within the turn limit","ok":true},{"name":"SLACK_SEND_MESSAGE completed once in the whole run","ok":true},{"name":"it ran on the consumer project, as consumer-user-e2e-acme-0001","ok":true},{"name":"it went to #general","ok":true},{"name":"its text was exactly \"build passed\"","ok":true},{"name":"no other tool that changes something ran","ok":true},{"name":"the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp","ok":true},{"name":"no account was connected outside the consumer project","ok":true}],"detours":[],"stop":"finished","modelTurns":4,"commands":[{"command":"composio run --help 2>&1 | head -80","exitCode":0},{"command":"composio search \"send slack message\" 2>&1 | head -40; composio execute SLACK_SEND_MESSAGE --get-schema 2>&1 | head -60","exitCode":0},{"command":"composio run '\nconst res = await execute(\"SLACK_SEND_MESSAGE\", { channel: \"general\", text: \"build passed\" });\nconst ts = res?.data?.ts ?? res?.data?.message?.ts ?? res?.ts ?? res?.message?.ts;\nconsole.log(\"timestamp:\", ts);\nconsole.log(JSON.stringify(res));\n' 2>&1","exitCode":0}],"costUsd":0.0012073650000000002,"durationMs":7179} |
| 50 | {"journey":"first-run-login","split":"train","arm":"no-skill","attempt":4,"fingerprint":"34bc8c4021a34a4f","outcome":"failed","checks":[{"name":"the model finished by itself, within the turn limit","ok":false,"seen":"\"{\\\"provider\\\":\\\"deadline\\\"}\""},{"name":"the first `composio whoami` reported no login","ok":true},{"name":"exactly one CLI login session was created, and it was linked","ok":false,"seen":"\"[{\\\"id\\\":\\\"00000000-0000-4000-8000-000000050001\\\",\\\"code\\\":\\\"E2E001\\\",\\\"status\\\":\\\"pending\\\",\\\"reportedLinked\\\":false}]\""},{"name":"the login completed: the user API key was fetched for the linked session","ok":false,"seen":"null"},{"name":"the login was stored: a new `composio whoami` exits 0 as the person, in their org","ok":false,"seen":"\"{\\\"exitCode\\\":1,\\\"stdout\\\":\\\"\\\",\\\"stderr\\\":\\\"You are not logged in yet. Please run `composio login`.\\\\n\\\"}\""},{"name":"no agent account was signed up in place of the person","ok":true}],"detours":[],"stop":"deadline","modelTurns":0,"commands":[{"command":"which composio && composio --version; composio whoami 2>&1 | head -50","exitCode":0},{"command":"composio login 2>&1 | head -60","exitCode":0}],"costUsd":0,"durationMs":360554} |