[mcp] Dedupe docs (#32929)
Previously the resource would return a bunch of dupes because the algolia results would return multiple hashes (headings) for the same url. --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/facebook/react/pull/32929). * #32932 * #32931 * #32930 * __->__ #32929 * #32928
lauren committed
Apr 16, 2025 at 17:48 UTC
fc21d5a7db4f714825111e365825804f478e093a
1 file changed
+21
-4
compiler/packages/react-mcp-server/src/index.ts
+21
-4
@@ -40,14 +40,30 @@ const server = new McpServer({
40
version: '0.0.0',
41
});
42
43
+function slugify(heading: string): string {
44
+ return heading
45
+ .split(' ')
46
+ .map(w => w.toLowerCase())
47
+ .join('-');
48
+}
49
+
50
// TODO: how to verify this works?
51
server.resource(
52
'docs',
53
new ResourceTemplate('docs://{message}', {list: undefined}),
47
- async (uri, {message}) => {
54
+ async (_uri, {message}) => {
55
const hits = await queryAlgolia(message);
56
+ const deduped = new Map();
57
+ for (const hit of hits) {
58
+ // drop hashes to dedupe properly
59
+ const u = new URL(hit.url);
60
+ if (deduped.has(u.pathname)) {
61
+ continue;
62
+ }
63
+ deduped.set(u.pathname, hit);
64
+ }
65
const pages: Array<string | null> = await Promise.all(
50
- hits.map(hit => {
66
+ Array.from(deduped.values()).map(hit => {
67
return fetch(hit.url, {
68
headers: {
69
'User-Agent':
@@ -70,16 +86,17 @@ server.resource(
86
.filter(html => html !== null)
87
.map(html => {
88
const $ = cheerio.load(html);
89
+ const title = encodeURIComponent(slugify($('h1').text()));
90
// react.dev should always have at least one <article> with the main content
91
const article = $('article').html();
92
if (article != null) {
93
return {
77
- uri: uri.href,
94
+ uri: `docs://${title}`,
95
text: turndownService.turndown(article),
96
};
97
} else {
98
return {
82
- uri: uri.href,
99
+ uri: `docs://${title}`,
100
// Fallback to converting the whole page to markdown
101
text: turndownService.turndown($.html()),
102
};