Implement generate-and-deploy workflow for GitHub Pages (#34)
* Add clickable repo links in weekly articles Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add weekly generate and deploy pipeline Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix PR #34 review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 18, 2026 at 13:47 UTC
3ffb5db6324fbba3ba3ff5a141f6f27c2fcaad0a
5 files changed
+298
.squad/agents/amy/history.md
+2
@@ -21,3 +21,5 @@
21
- **2026-05-18T10:27:35.339+02:00:** Weekly content generation should follow the `archetypes/weekly.md` frontmatter schema: `title`, `date`, `week`, `tags`, `categories`, `repos_featured`, `stars_tracked`, `top_repo`, and `summary`, followed by the five standard analysis sections.
22
- **2026-05-18T10:27:35.339+02:00:** RSS is enabled through Hugo outputs in `hugo.toml`, and the verified root feed for this setup is `public/index.xml` alongside section and taxonomy RSS pages.
23
- **2026-05-18T10:59:10Z:** Issues #3 and #4 complete. Commits c46beab, 05372c3. Ready for Issue #6+. User directive: all future work follows branch → PR → Review → Merge workflow (no direct commits to main).
24
+- **2026-05-18T13:20:07.067+02:00:** The weekly pipeline now needs three repo-backed publication stages after crawling: analyzed markdown in `data/analyzed/`, generated Hugo content in `content/weekly/YYYY/WNN.md`, and a Pages artifact built with Hugo 0.161.1 plus Pagefind.
25
+- **2026-05-18T13:20:07.067+02:00:** `scripts/generate_content.py` can safely normalize analyzer output into archetype-compatible Hugo frontmatter by dropping analyzer-only fields (`year`, `quality_score`) and keeping the analysis body intact for publication.
.squad/decisions/inbox/amy-generate-deploy.md
new
+15
@@ -0,0 +1,15 @@
1
+# Amy decision inbox — generate and deploy workflow
2
+
3
+- **Date:** 2026-05-18T13:20:07.067+02:00
4
+- **Issue:** #11 — Implement generate-and-deploy workflow for GitHub Pages
5
+
6
+## Proposed decision
7
+
8
+Keep `.github/workflows/deploy-site.yml` for push-to-main deployments, and let `.github/workflows/crawl-and-publish.yml` own the weekly automation path end-to-end.
9
+
10
+### Implementation details
11
+
12
+1. The weekly workflow should run `crawl → analyze → generate → deploy` in a single pipeline.
13
+2. The `generate` stage should write the weekly Hugo page into `content/weekly/YYYY/WNN.md` using archetype-compatible frontmatter derived from `data/analyzed/YYYY-WNN-summary.md`.
14
+3. The generated weekly page should be committed back to the default branch before the Pages build so future archive builds retain previously published weekly content.
15
+4. The publish artifact should be built with Hugo 0.161.1 and Pagefind, then deployed with `actions/deploy-pages@v4` under the `github-pages` environment.
README.md
+2
@@ -37,3 +37,5 @@ SquadScope is a Hugo-powered GitHub Pages site for weekly, monthly, and yearly t
37
## Deployment
38
39
Pushing to `main` triggers `.github/workflows/deploy-site.yml`, which builds the Hugo site and deploys the generated `public/` directory to GitHub Pages.
40
+
41
+The scheduled weekly pipeline in `.github/workflows/crawl-and-publish.yml` now runs `crawl → analyze → generate → deploy`, using `scripts/generate_content.py` to turn `data/analyzed/YYYY-WNN-summary.md` into `content/weekly/YYYY/WNN.md` before the Pages build and Pagefind indexing steps.
scripts/generate_content.py
new
+196
@@ -0,0 +1,196 @@
1
+from __future__ import annotations
2
+
3
+import argparse
4
+import csv
5
+import re
6
+from pathlib import Path
7
+
8
+FRONTMATTER_PATTERN = re.compile(r"^---\n(.*?)\n---\n(.*)\Z", re.DOTALL)
9
+WEEK_PATTERN = re.compile(r"^(?P<year>\d{4})-W(?P<week>\d{2})$")
10
+SUMMARY_SUFFIX = "-summary.md"
11
+ANALYSIS_SUFFIX = " Analysis"
12
+REQUIRED_ANALYSIS_FIELDS = {
13
+ "title",
14
+ "date",
15
+ "week",
16
+ "year",
17
+ "tags",
18
+ "categories",
19
+ "repos_featured",
20
+ "stars_tracked",
21
+ "top_repo",
22
+ "quality_score",
23
+ "summary",
24
+}
25
+
26
+
27
+class GenerationError(ValueError):
28
+ pass
29
+
30
+
31
+def parse_args() -> argparse.Namespace:
32
+ parser = argparse.ArgumentParser(
33
+ description="Generate a Hugo weekly content page from an analyzed summary markdown file."
34
+ )
35
+ parser.add_argument(
36
+ "summary",
37
+ nargs="?",
38
+ default=None,
39
+ help="Path to data/analyzed/YYYY-WNN-summary.md. Defaults to the newest analyzed summary.",
40
+ )
41
+ parser.add_argument(
42
+ "--output",
43
+ default=None,
44
+ help="Optional explicit output path. Defaults to content/weekly/YYYY/WNN.md.",
45
+ )
46
+ return parser.parse_args()
47
+
48
+
49
+def parse_week(value: str) -> tuple[int, int]:
50
+ match = WEEK_PATTERN.fullmatch(value)
51
+ if not match:
52
+ raise GenerationError(f"Invalid week value: {value}")
53
+ return int(match.group("year")), int(match.group("week"))
54
+
55
+
56
+def week_from_summary_path(path: Path) -> tuple[int, int]:
57
+ if not path.name.endswith(SUMMARY_SUFFIX):
58
+ raise GenerationError(f"Invalid summary filename: {path.name}")
59
+ return parse_week(path.name.removesuffix(SUMMARY_SUFFIX))
60
+
61
+
62
+def find_latest_summary(root: Path) -> Path:
63
+ candidates = list(root.glob(f"data/analyzed/*{SUMMARY_SUFFIX}"))
64
+ if not candidates:
65
+ raise GenerationError("No analyzed summaries found under data/analyzed/.")
66
+ return max(candidates, key=week_from_summary_path)
67
+
68
+
69
+def parse_scalar(value: str):
70
+ value = value.strip()
71
+ if not value:
72
+ return ""
73
+ if value.startswith("[") and value.endswith("]"):
74
+ inner = value[1:-1].strip()
75
+ if not inner:
76
+ return []
77
+ return [item.strip().strip('"').strip("'") for item in csv.reader([inner], skipinitialspace=True).__next__()]
78
+ if value.startswith(('"', "'")) and value.endswith(('"', "'")):
79
+ return value[1:-1]
80
+ if re.fullmatch(r"-?\d+", value):
81
+ return int(value)
82
+ if value.lower() == "true":
83
+ return True
84
+ if value.lower() == "false":
85
+ return False
86
+ return value
87
+
88
+
89
+def parse_frontmatter(document: str) -> tuple[dict[str, object], str]:
90
+ match = FRONTMATTER_PATTERN.match(document)
91
+ if not match:
92
+ raise GenerationError("Summary is missing YAML frontmatter.")
93
+
94
+ frontmatter_text, body = match.groups()
95
+ frontmatter: dict[str, object] = {}
96
+ for line in frontmatter_text.splitlines():
97
+ if not line.strip():
98
+ continue
99
+ if ":" not in line:
100
+ raise GenerationError(f"Malformed frontmatter line: {line}")
101
+ key, raw_value = line.split(":", 1)
102
+ frontmatter[key.strip()] = parse_scalar(raw_value)
103
+
104
+ missing = REQUIRED_ANALYSIS_FIELDS.difference(frontmatter)
105
+ if missing:
106
+ raise GenerationError(f"Missing required analysis fields: {', '.join(sorted(missing))}")
107
+
108
+ return frontmatter, body.strip() + "\n"
109
+
110
+
111
+def normalize_title(title: str) -> str:
112
+ if title.endswith(ANALYSIS_SUFFIX):
113
+ return title[: -len(ANALYSIS_SUFFIX)]
114
+ return title
115
+
116
+
117
+def ensure_list(value: object, *, field_name: str) -> list[str]:
118
+ if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value):
119
+ raise GenerationError(f"{field_name} must be a non-empty list of strings.")
120
+ return value
121
+
122
+
123
+def infer_output_path(week: str, root: Path) -> Path:
124
+ year, week_number = parse_week(week)
125
+ return root / "content" / "weekly" / str(year) / f"W{week_number:02d}.md"
126
+
127
+
128
+def yaml_quote(value: str) -> str:
129
+ return '"' + value.replace('\\', '\\\\').replace('"', '\\"') + '"'
130
+
131
+
132
+def render_frontmatter(data: dict[str, object]) -> str:
133
+ lines = [
134
+ "---",
135
+ f'title: {yaml_quote(str(data["title"]))}',
136
+ f'date: {data["date"]}',
137
+ f'week: {yaml_quote(str(data["week"]))}',
138
+ f'tags: [{", ".join(data["tags"])}]',
139
+ f'categories: [{", ".join(data["categories"])}]',
140
+ f'repos_featured: {data["repos_featured"]}',
141
+ f'stars_tracked: {data["stars_tracked"]}',
142
+ f'top_repo: {yaml_quote(str(data["top_repo"]))}',
143
+ f'summary: {yaml_quote(str(data["summary"]))}',
144
+ "draft: false",
145
+ "---",
146
+ "",
147
+ ]
148
+ return "\n".join(lines)
149
+
150
+
151
+def transform_summary(frontmatter: dict[str, object], body: str) -> str:
152
+ tags = ensure_list(frontmatter["tags"], field_name="tags")
153
+ categories = ensure_list(frontmatter["categories"], field_name="categories")
154
+ if "weekly" not in categories:
155
+ categories = [*categories, "weekly"]
156
+
157
+ page_frontmatter = {
158
+ "title": normalize_title(str(frontmatter["title"])),
159
+ "date": str(frontmatter["date"]),
160
+ "week": str(frontmatter["week"]),
161
+ "tags": tags,
162
+ "categories": categories,
163
+ "repos_featured": int(frontmatter["repos_featured"]),
164
+ "stars_tracked": int(frontmatter["stars_tracked"]),
165
+ "top_repo": str(frontmatter["top_repo"]),
166
+ "summary": str(frontmatter["summary"]),
167
+ }
168
+ return render_frontmatter(page_frontmatter) + "\n" + body.lstrip()
169
+
170
+
171
+def generate_content(summary_path: Path, output_path: Path | None = None) -> Path:
172
+ root = Path.cwd()
173
+ document = summary_path.read_text(encoding="utf-8")
174
+ frontmatter, body = parse_frontmatter(document)
175
+ target_path = output_path or infer_output_path(str(frontmatter["week"]), root)
176
+ target_path.parent.mkdir(parents=True, exist_ok=True)
177
+ target_path.write_text(transform_summary(frontmatter, body), encoding="utf-8")
178
+ return target_path
179
+
180
+
181
+def main() -> int:
182
+ args = parse_args()
183
+ root = Path.cwd()
184
+ summary_path = Path(args.summary) if args.summary else find_latest_summary(root)
185
+ output_path = Path(args.output) if args.output else None
186
+
187
+ if not summary_path.exists():
188
+ raise SystemExit(f"Summary file not found: {summary_path}")
189
+
190
+ written_path = generate_content(summary_path, output_path)
191
+ print(f"Generated {written_path} from {summary_path}")
192
+ return 0
193
+
194
+
195
+if __name__ == "__main__":
196
+ raise SystemExit(main())
tests/test_generate_content.py
new
+83
@@ -0,0 +1,83 @@
1
+import tempfile
2
+import unittest
3
+from pathlib import Path
4
+
5
+import scripts.generate_content as generate_content
6
+
7
+
8
+class GenerateContentTests(unittest.TestCase):
9
+ def test_generate_content_creates_hugo_weekly_page(self) -> None:
10
+ tests_root = Path(__file__).resolve().parent
11
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
12
+ base = Path(tmpdir)
13
+ summary_path = base / "data" / "analyzed" / "2026-W21-summary.md"
14
+ summary_path.parent.mkdir(parents=True)
15
+ summary_path.write_text(
16
+ """---
17
+title: \"Week 21, 2026 Analysis\"
18
+date: 2026-05-18T13:20:07.067+02:00
19
+week: \"2026-W21\"
20
+year: 2026
21
+tags: [ai, agents, developer-tooling]
22
+categories: [weekly]
23
+repos_featured: 12
24
+stars_tracked: 3456
25
+top_repo: \"octo/repo\"
26
+quality_score: 88
27
+summary: \"Agent tooling became more operational this week.\"
28
+---
29
+
30
+## Notable New Repositories
31
+
32
+Body copy.
33
+""",
34
+ encoding="utf-8",
35
+ )
36
+
37
+ previous_cwd = Path.cwd()
38
+ try:
39
+ import os
40
+
41
+ os.chdir(base)
42
+ output_path = generate_content.generate_content(summary_path)
43
+ finally:
44
+ os.chdir(previous_cwd)
45
+
46
+ self.assertEqual(output_path, base / "content" / "weekly" / "2026" / "W21.md")
47
+ rendered = output_path.read_text(encoding="utf-8")
48
+ self.assertIn('title: "Week 21, 2026"', rendered)
49
+ self.assertIn("draft: false", rendered)
50
+ self.assertIn('week: "2026-W21"', rendered)
51
+ self.assertIn('top_repo: "octo/repo"', rendered)
52
+ self.assertIn('summary: "Agent tooling became more operational this week."', rendered)
53
+ self.assertNotIn("quality_score", rendered)
54
+ self.assertNotIn("year:", rendered)
55
+ self.assertIn("## Notable New Repositories", rendered)
56
+
57
+ def test_find_latest_summary_uses_week_not_mtime(self) -> None:
58
+ tests_root = Path(__file__).resolve().parent
59
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
60
+ base = Path(tmpdir)
61
+ analyzed_dir = base / "data" / "analyzed"
62
+ analyzed_dir.mkdir(parents=True)
63
+ latest = analyzed_dir / "2027-W01-summary.md"
64
+ older = analyzed_dir / "2026-W52-summary.md"
65
+ latest.write_text("latest\n", encoding="utf-8")
66
+ older.write_text("older but touched later\n", encoding="utf-8")
67
+
68
+ self.assertEqual(generate_content.find_latest_summary(base), latest)
69
+
70
+ def test_parse_frontmatter_rejects_missing_required_fields(self) -> None:
71
+ with self.assertRaises(generate_content.GenerationError):
72
+ generate_content.parse_frontmatter(
73
+ """---
74
+title: \"Week 21, 2026 Analysis\"
75
+---
76
+
77
+## Notable New Repositories
78
+"""
79
+ )
80
+
81
+
82
+if __name__ == "__main__":
83
+ unittest.main()