feat: namespace data directories by topic ID (#60) (#94)
Add scripts/topic_paths.py as single source of truth for data directory layout. Paths resolve to data/{type}/{topic_id}/ when a topic is given, or the legacy flat data/{type}/ when topic is None or 'general'. - Add --topic flag to crawl.py - Update generate_content.py to use topic-aware paths - 19 new tests for topic_paths module - Backward compatible with existing flat layout Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 19, 2026 at 15:41 UTC
cfc2bf09225570bca26463cc4aa5345432257003
27 files changed
+221
-11
.gitignore
+1
@@ -15,3 +15,4 @@ resources/_gen/
15
16
# Git worktrees
17
.worktrees/
18
+*.pyc
data/snapshots/2026-W21-stars.json
new
+9
@@ -0,0 +1,9 @@
1
+{
2
+ "week": "2026-W21",
3
+ "captured_at": "2026-05-18T08:00:00Z",
4
+ "repository_count": 2,
5
+ "stars": {
6
+ "octo/momentum-watch": 180,
7
+ "octo/signal-kit": 120
8
+ }
9
+}
scripts/__pycache__/__init__.cpython-312.pyc
Binary files a/scripts/__pycache__/__init__.cpython-312.pyc and /dev/null differ
scripts/__pycache__/analysis_gate.cpython-312.pyc
Binary files a/scripts/__pycache__/analysis_gate.cpython-312.pyc and /dev/null differ
scripts/__pycache__/analyze_fallback.cpython-312.pyc
Binary files a/scripts/__pycache__/analyze_fallback.cpython-312.pyc and /dev/null differ
scripts/__pycache__/crawl.cpython-312.pyc
Binary files a/scripts/__pycache__/crawl.cpython-312.pyc and /dev/null differ
scripts/__pycache__/generate_content.cpython-312.pyc
Binary files a/scripts/__pycache__/generate_content.cpython-312.pyc and /dev/null differ
scripts/__pycache__/generate_rollups.cpython-312.pyc
Binary files a/scripts/__pycache__/generate_rollups.cpython-312.pyc and /dev/null differ
scripts/__pycache__/reskill.cpython-312.pyc
Binary files a/scripts/__pycache__/reskill.cpython-312.pyc and /dev/null differ
scripts/__pycache__/track_quality.cpython-312.pyc
Binary files a/scripts/__pycache__/track_quality.cpython-312.pyc and /dev/null differ
scripts/__pycache__/track_token_usage.cpython-312.pyc
Binary files a/scripts/__pycache__/track_token_usage.cpython-312.pyc and /dev/null differ
scripts/crawl.py
+15
-3
@@ -18,6 +18,8 @@ from pathlib import Path
18
from typing import Any, Iterable
19
from urllib import error, parse, request
20
21
+from scripts.topic_paths import cache_dir, raw_dir, snapshots_dir
22
+
23
API_ROOT = "https://api.github.com"
24
SEARCH_REPOSITORIES = f"{API_ROOT}/search/repositories"
25
CACHE_ROOT = Path("data/cache")
@@ -477,6 +479,11 @@ def parse_args() -> argparse.Namespace:
479
"--output",
480
help="Optional explicit output path. Defaults to data/raw/YYYY-WNN.json.",
481
)
482
+ parser.add_argument(
483
+ "--topic",
484
+ default=None,
485
+ help="Topic ID for namespaced data directories. Defaults to 'general' (flat layout).",
486
+ )
487
return parser.parse_args()
488
489
@@ -744,13 +751,18 @@ def main() -> int:
751
print("GITHUB_TOKEN is required", file=sys.stderr)
752
return 1
753
754
+ topic_id = args.topic
755
+ topic_raw = raw_dir(topic_id)
756
+ topic_snapshots = snapshots_dir(topic_id)
757
+ topic_cache = cache_dir(topic_id)
758
+
759
crawled_at = utc_now()
760
window_end = datetime.strptime(args.as_of, "%Y-%m-%d").replace(tzinfo=UTC) if args.as_of else crawled_at
761
since = datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=UTC) if args.since else window_end - timedelta(days=7)
762
week = week_slug(window_end)
751
- output_path = Path(args.output) if args.output else RAW_ROOT / f"{week}.json"
752
- snapshot_path = SNAPSHOT_ROOT / f"{week}-stars.json"
753
- client = GitHubClient(github_token)
763
+ output_path = Path(args.output) if args.output else topic_raw / f"{week}.json"
764
+ snapshot_path = topic_snapshots / f"{week}-stars.json"
765
+ client = GitHubClient(github_token, cache_dir=topic_cache)
766
max_results = max(1, min(args.max_results, 1000))
767
768
if args.as_of:
scripts/generate_content.py
+8
-2
@@ -5,6 +5,8 @@ import csv
5
import re
6
from pathlib import Path
7
8
+from scripts.topic_paths import analyzed_dir
9
+
10
FRONTMATTER_PATTERN = re.compile(r"^---\n(.*?)\n---\n(.*)\Z", re.DOTALL)
11
WEEK_PATTERN = re.compile(r"^(?P<year>\d{4})-W(?P<week>\d{2})$")
12
SUMMARY_SUFFIX = "-summary.md"
@@ -59,8 +61,12 @@ def week_from_summary_path(path: Path) -> tuple[int, int]:
61
return parse_week(path.name.removesuffix(SUMMARY_SUFFIX))
62
63
62
-def find_latest_summary(root: Path) -> Path:
63
- candidates = list(root.glob(f"data/analyzed/*{SUMMARY_SUFFIX}"))
64
+def find_latest_summary(root: Path, topic_id: str | None = None) -> Path:
65
+ search_dir = analyzed_dir(topic_id)
66
+ candidates = list(search_dir.glob(f"*{SUMMARY_SUFFIX}"))
67
+ if not candidates:
68
+ # Fallback: try legacy path via root
69
+ candidates = list(root.glob(f"data/analyzed/*{SUMMARY_SUFFIX}"))
70
if not candidates:
71
raise GenerationError("No analyzed summaries found under data/analyzed/.")
72
return max(candidates, key=week_from_summary_path)
scripts/topic_paths.py
new
+76
@@ -0,0 +1,76 @@
1
+#!/usr/bin/env python3
2
+"""Topic-aware data path resolution for SquadScope.
3
+
4
+Provides a single source of truth for data directory layout:
5
+ data/raw/{topic_id}/
6
+ data/analyzed/{topic_id}/
7
+ data/metrics/{topic_id}/
8
+ data/snapshots/{topic_id}/
9
+ data/cache/{topic_id}/
10
+
11
+Backward compatibility: when topic_id is None or "general", paths
12
+resolve to the legacy flat layout (data/raw/, data/analyzed/, etc.).
13
+"""
14
+
15
+from __future__ import annotations
16
+
17
+from pathlib import Path
18
+
19
+DEFAULT_TOPIC = "general"
20
+
21
+# Base data root (relative to repo root)
22
+DATA_ROOT = Path("data")
23
+
24
+
25
+def _resolve(base: Path, topic_id: str | None) -> Path:
26
+ """Return namespaced path, creating parents on first access."""
27
+ tid = (topic_id or DEFAULT_TOPIC).strip().lower()
28
+ if tid == DEFAULT_TOPIC:
29
+ return base
30
+ return base / tid
31
+
32
+
33
+def raw_dir(topic_id: str | None = None) -> Path:
34
+ """Return the raw data directory for a given topic."""
35
+ return _resolve(DATA_ROOT / "raw", topic_id)
36
+
37
+
38
+def analyzed_dir(topic_id: str | None = None) -> Path:
39
+ """Return the analyzed data directory for a given topic."""
40
+ return _resolve(DATA_ROOT / "analyzed", topic_id)
41
+
42
+
43
+def metrics_dir(topic_id: str | None = None) -> Path:
44
+ """Return the metrics directory for a given topic."""
45
+ return _resolve(DATA_ROOT / "metrics", topic_id)
46
+
47
+
48
+def snapshots_dir(topic_id: str | None = None) -> Path:
49
+ """Return the snapshots directory for a given topic."""
50
+ return _resolve(DATA_ROOT / "snapshots", topic_id)
51
+
52
+
53
+def cache_dir(topic_id: str | None = None) -> Path:
54
+ """Return the cache directory for a given topic."""
55
+ return _resolve(DATA_ROOT / "cache", topic_id)
56
+
57
+
58
+def ensure_dirs(topic_id: str | None = None) -> None:
59
+ """Create all data directories for a topic if they don't exist."""
60
+ for fn in (raw_dir, analyzed_dir, metrics_dir, snapshots_dir, cache_dir):
61
+ fn(topic_id).mkdir(parents=True, exist_ok=True)
62
+
63
+
64
+def load_topic_id(config_path: str | Path = "squadscope.topic.yml") -> str:
65
+ """Load topic.id from a YAML config file. Returns DEFAULT_TOPIC on failure."""
66
+ import yaml
67
+
68
+ path = Path(config_path)
69
+ if not path.exists():
70
+ return DEFAULT_TOPIC
71
+ try:
72
+ with open(path, encoding="utf-8") as f:
73
+ data = yaml.safe_load(f)
74
+ return (data or {}).get("topic", {}).get("id", DEFAULT_TOPIC)
75
+ except Exception:
76
+ return DEFAULT_TOPIC
tests/__pycache__/test_analysis_gate.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_analysis_gate.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_analyze_fallback.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_analyze_fallback.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_crawl.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_crawl.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_generate_content.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_generate_content.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_generate_rollups.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_generate_rollups.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_pipeline.cpython-312-pytest-9.0.2.pyc
Binary files a/tests/__pycache__/test_pipeline.cpython-312-pytest-9.0.2.pyc and /dev/null differ
tests/__pycache__/test_pipeline.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_pipeline.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_reskill.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_reskill.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_track_quality.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_track_quality.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/__pycache__/test_track_token_usage.cpython-312-pytest-9.0.3.pyc
Binary files a/tests/__pycache__/test_track_token_usage.cpython-312-pytest-9.0.3.pyc and /dev/null differ
tests/test_crawl.py
+4
-4
@@ -100,7 +100,7 @@ class CrawlTests(unittest.TestCase):
100
queries: list[str] = []
101
102
class FakeClient:
103
- def __init__(self, token: str) -> None:
103
+ def __init__(self, token: str, **kwargs) -> None:
104
self.token = token
105
self.api_calls_used = 0
106
self.cache_hits = 0
@@ -118,7 +118,7 @@ class CrawlTests(unittest.TestCase):
118
def has_readme(self, full_name: str) -> bool:
119
return True
120
121
- args = Namespace(since="2026-05-11", as_of=None, max_results=25, output="data/raw/test-live.json")
121
+ args = Namespace(since="2026-05-11", as_of=None, max_results=25, output="data/raw/test-live.json", topic=None)
122
with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(
123
"os.environ", {"GITHUB_TOKEN": "token"}, clear=False
124
), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(
@@ -133,7 +133,7 @@ class CrawlTests(unittest.TestCase):
133
queries: list[str] = []
134
135
class FakeClient:
136
- def __init__(self, token: str) -> None:
136
+ def __init__(self, token: str, **kwargs) -> None:
137
self.token = token
138
self.api_calls_used = 0
139
self.cache_hits = 0
@@ -151,7 +151,7 @@ class CrawlTests(unittest.TestCase):
151
def has_readme(self, full_name: str) -> bool:
152
return True
153
154
- args = Namespace(since="2026-05-11", as_of="2026-05-18", max_results=25, output="data/raw/test-backfill.json")
154
+ args = Namespace(since="2026-05-11", as_of="2026-05-18", max_results=25, output="data/raw/test-backfill.json", topic=None)
155
with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(
156
"os.environ", {"GITHUB_TOKEN": "token"}, clear=False
157
), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(
tests/test_pipeline.py
+3
-2
@@ -288,7 +288,7 @@ class PipelineIntegrationTests(unittest.TestCase):
288
)
289
290
class FakeClient:
291
- def __init__(self, token: str) -> None:
291
+ def __init__(self, token: str, **kwargs) -> None:
292
self.token = token
293
self.api_calls_used = 2
294
self.cache_hits = 1
@@ -314,6 +314,7 @@ class PipelineIntegrationTests(unittest.TestCase):
314
as_of="2026-05-18",
315
max_results=10,
316
output=str(output_path),
317
+ topic=None,
318
)
319
320
with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(
@@ -321,7 +322,7 @@ class PipelineIntegrationTests(unittest.TestCase):
322
), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(
323
crawl, "load_previous_star_snapshot", return_value={"octo/momentum-watch": 145}
324
), mock.patch.object(crawl, "utc_now", return_value=FIXED_RUN_TIME), mock.patch.object(
324
- crawl, "SNAPSHOT_ROOT", snapshot_dir
325
+ crawl, "snapshots_dir", return_value=snapshot_dir
326
):
327
exit_code = crawl.main()
328
tests/test_topic_paths.py
new
+105
@@ -0,0 +1,105 @@
1
+"""Tests for scripts/topic_paths.py — topic-aware data directory resolution."""
2
+
3
+from __future__ import annotations
4
+
5
+import tempfile
6
+from pathlib import Path
7
+
8
+import pytest
9
+
10
+from scripts.topic_paths import (
11
+ DEFAULT_TOPIC,
12
+ DATA_ROOT,
13
+ analyzed_dir,
14
+ cache_dir,
15
+ ensure_dirs,
16
+ load_topic_id,
17
+ metrics_dir,
18
+ raw_dir,
19
+ snapshots_dir,
20
+)
21
+
22
+
23
+class TestDefaultTopic:
24
+ """When topic is None or 'general', use legacy flat paths."""
25
+
26
+ def test_raw_dir_none(self):
27
+ assert raw_dir(None) == DATA_ROOT / "raw"
28
+
29
+ def test_raw_dir_general(self):
30
+ assert raw_dir("general") == DATA_ROOT / "raw"
31
+
32
+ def test_analyzed_dir_default(self):
33
+ assert analyzed_dir() == DATA_ROOT / "analyzed"
34
+
35
+ def test_metrics_dir_default(self):
36
+ assert metrics_dir() == DATA_ROOT / "metrics"
37
+
38
+ def test_snapshots_dir_default(self):
39
+ assert snapshots_dir() == DATA_ROOT / "snapshots"
40
+
41
+ def test_cache_dir_default(self):
42
+ assert cache_dir() == DATA_ROOT / "cache"
43
+
44
+
45
+class TestNamespacedTopic:
46
+ """When a specific topic is given, paths include the topic subdirectory."""
47
+
48
+ def test_raw_dir_ai_ml(self):
49
+ assert raw_dir("ai-ml") == DATA_ROOT / "raw" / "ai-ml"
50
+
51
+ def test_analyzed_dir_rust(self):
52
+ assert analyzed_dir("rust") == DATA_ROOT / "analyzed" / "rust"
53
+
54
+ def test_metrics_dir_topic(self):
55
+ assert metrics_dir("web-dev") == DATA_ROOT / "metrics" / "web-dev"
56
+
57
+ def test_snapshots_dir_topic(self):
58
+ assert snapshots_dir("ai-ml") == DATA_ROOT / "snapshots" / "ai-ml"
59
+
60
+ def test_cache_dir_topic(self):
61
+ assert cache_dir("rust") == DATA_ROOT / "cache" / "rust"
62
+
63
+ def test_case_normalization(self):
64
+ assert raw_dir("AI-ML") == DATA_ROOT / "raw" / "ai-ml"
65
+
66
+ def test_whitespace_stripped(self):
67
+ assert raw_dir(" rust ") == DATA_ROOT / "raw" / "rust"
68
+
69
+
70
+class TestEnsureDirs:
71
+ """ensure_dirs creates all required directories."""
72
+
73
+ def test_creates_all_dirs(self, tmp_path, monkeypatch):
74
+ monkeypatch.setattr("scripts.topic_paths.DATA_ROOT", tmp_path / "data")
75
+ ensure_dirs("ai-ml")
76
+ for subdir in ("raw", "analyzed", "metrics", "snapshots", "cache"):
77
+ assert (tmp_path / "data" / subdir / "ai-ml").is_dir()
78
+
79
+ def test_creates_flat_dirs_for_general(self, tmp_path, monkeypatch):
80
+ monkeypatch.setattr("scripts.topic_paths.DATA_ROOT", tmp_path / "data")
81
+ ensure_dirs(None)
82
+ for subdir in ("raw", "analyzed", "metrics", "snapshots", "cache"):
83
+ assert (tmp_path / "data" / subdir).is_dir()
84
+
85
+
86
+class TestLoadTopicId:
87
+ """load_topic_id reads the topic.id from YAML config."""
88
+
89
+ def test_reads_valid_config(self, tmp_path):
90
+ config = tmp_path / "topic.yml"
91
+ config.write_text("topic:\n id: rust\n name: Rust\n")
92
+ assert load_topic_id(config) == "rust"
93
+
94
+ def test_missing_file_returns_default(self, tmp_path):
95
+ assert load_topic_id(tmp_path / "nope.yml") == DEFAULT_TOPIC
96
+
97
+ def test_malformed_yaml_returns_default(self, tmp_path):
98
+ config = tmp_path / "bad.yml"
99
+ config.write_text("{{invalid yaml")
100
+ assert load_topic_id(config) == DEFAULT_TOPIC
101
+
102
+ def test_missing_topic_key_returns_default(self, tmp_path):
103
+ config = tmp_path / "empty.yml"
104
+ config.write_text("scoring:\n min_stars: 10\n")
105
+ assert load_topic_id(config) == DEFAULT_TOPIC