Don't crawl /repos and /irc
This pollutes the search function.
Eelco Dolstra committed
Oct 15, 2014 at 11:49 UTC
14c59a3f532c0909618d7029e0e569231b1374ec
1 file changed
+7
-2
nixos-org/webserver.nix
+7
-2
@@ -30,13 +30,18 @@ let
30
{ urlPath = "/nixos/manual-raw";
31
dir = "/releases/channels/nixos-14.04/manual";
32
}
33
- /*
33
{ urlPath = "/new";
34
dir = "/home/eelco/nixos-homepage-new";
35
}
37
- */
36
];
37
38
+ robotsEntries =
39
+ ''
40
+ User-agent: *
41
+ Disallow: /repos/
42
+ Disallow: /irc/
43
+ '';
44
+
45
extraConfig =
46
''
47
MaxKeepAliveRequests 0