| 1 | import sys |
| 2 | from pathlib import Path |
| 3 | |
| 4 | |
| 5 | PROJECT_ROOT = Path(__file__).resolve().parents[1] |
| 6 | if str(PROJECT_ROOT) not in sys.path: |
| 7 | sys.path.insert(0, str(PROJECT_ROOT)) |
| 8 | |
| 9 | |
| 10 | from plugins._plugin_scan.helpers.prompt import build_prompt |
| 11 | |
| 12 | |
| 13 | def test_plugin_scan_prompt_calibrates_provider_credentials_as_expected_behavior(): |
| 14 | prompt = build_prompt( |
| 15 | "https://github.com/example/provider-plugin", |
| 16 | checks=["remoteComms", "secrets"], |
| 17 | ) |
| 18 | |
| 19 | assert "Classify by demonstrated risk, not by the mere presence of a capability." in prompt |
| 20 | assert "adds `conf/model_providers.yaml`" in prompt |
| 21 | assert "provider credentials when those capabilities are transparent" in prompt |
| 22 | assert ( |
| 23 | "No secret access, or credential access is narrow, user-supplied/provider-specific, " |
| 24 | "and justified by the declared purpose" |
| 25 | ) in prompt |
| 26 | assert ( |
| 27 | "No network calls, or network calls are transparent and necessary for the declared plugin purpose" |
| 28 | in prompt |
| 29 | ) |
| 30 | |
| 31 | |
| 32 | def test_plugin_scan_prompt_calibrates_common_plugin_capabilities_as_expected_behavior(): |
| 33 | prompt = build_prompt( |
| 34 | "https://github.com/example/plugin", |
| 35 | checks=["structure", "codeReview", "agentManipulation", "obfuscation"], |
| 36 | ) |
| 37 | |
| 38 | assert ( |
| 39 | "A plugin can legitimately add tools, prompts, API handlers, hooks, settings, " |
| 40 | "dependencies, scheduled jobs, network clients, subprocess calls, filesystem access" |
| 41 | ) in prompt |
| 42 | assert "Treat expected filesystem access as 🟢" in prompt |
| 43 | assert "Treat expected subprocess use as 🟢" in prompt |
| 44 | assert "Treat prompt/system text as 🟢" in prompt |
| 45 | assert "ordinary minified/vendor/generated frontend assets" in prompt |
| 46 | assert "normal plugin prompt templates, tool instructions, README usage examples" in prompt |
| 47 | assert "fixed subprocess commands, dependency installation hooks" in prompt |
| 48 | |
| 49 | |
| 50 | def test_plugin_scan_prompt_requires_file_paths_only_for_warning_or_fail_findings(): |
| 51 | prompt = build_prompt( |
| 52 | "https://github.com/example/provider-plugin", |
| 53 | checks=["secrets"], |
| 54 | ) |
| 55 | |
| 56 | assert "Every 🟡 or 🔴 finding has a concrete file path and line range" in prompt |
| 57 | assert "Expected plugin capabilities were not treated as findings" in prompt |
| 58 | assert "Each check has a concrete finding with file path" not in prompt |