feat: Add `noindex` option to generate `robots.txt` disallowing crawlers and ignore log files.
lemon-mint committed
Nov 20, 2025 at 09:54 UTC
88371b65c62f7ec8289ae0a5964217e8a069b410
4 files changed
+17
-2
.gitignore
+2
@@ -38,3 +38,5 @@ profile.cov
38
# Docker Compose Override
39
docker-compose.override.yml
40
docker-compose.override.yaml
41
+
42
+*.log
\ No newline at end of file
Dockerfile
+1
@@ -28,6 +28,7 @@ COPY --from=builder /src/bin/relay-server /usr/bin/relay-server
28
ENV PORTAL_URL=http://localhost:4017
29
ENV PORTAL_APP_URL=http://*.localhost:4017
30
ENV BOOTSTRAP_URIS=ws://localhost:4017/relay
31
+ENV NOINDEX=false
32
33
EXPOSE 4017
34
ENTRYPOINT ["/usr/bin/relay-server"]
cmd/relay-server/main.go
+5
-1
@@ -26,6 +26,7 @@ var (
26
flagPort int
27
flagMaxLease int
28
flagLeaseBPS int
29
+ flagNoIndex bool
30
)
31
32
func main() {
@@ -53,6 +54,9 @@ func main() {
54
flag.IntVar(&flagPort, "port", 4017, "app UI and HTTP proxy port")
55
flag.IntVar(&flagMaxLease, "max-lease", 0, "maximum active relayed connections per lease (0 = unlimited)")
56
flag.IntVar(&flagLeaseBPS, "lease-bps", 0, "default bytes-per-second limit per lease (0 = unlimited)")
57
+
58
+ defaultNoIndex := os.Getenv("NOINDEX") == "true"
59
+ flag.BoolVar(&flagNoIndex, "noindex", defaultNoIndex, "disallow all crawlers via robots.txt (env: NOINDEX)")
60
flag.Parse()
61
62
flagBootstraps = utils.ParseURLs(flagBootstrapsCSV)
@@ -83,7 +87,7 @@ func runServer() error {
87
serv.Start()
88
defer serv.Stop()
89
86
- httpSrv := serveHTTP(fmt.Sprintf(":%d", flagPort), serv, cred.ID(), flagBootstraps, stop)
90
+ httpSrv := serveHTTP(fmt.Sprintf(":%d", flagPort), serv, cred.ID(), flagBootstraps, flagNoIndex, stop)
91
92
<-ctx.Done()
93
log.Info().Msg("[server] shutting down...")
cmd/relay-server/view.go
+9
-1
@@ -20,7 +20,7 @@ import (
20
var distFS embed.FS
21
22
// serveHTTP builds the HTTP mux and returns the server.
23
-func serveHTTP(addr string, serv *portal.RelayServer, nodeID string, bootstraps []string, cancel context.CancelFunc) *http.Server {
23
+func serveHTTP(addr string, serv *portal.RelayServer, nodeID string, bootstraps []string, noIndex bool, cancel context.CancelFunc) *http.Server {
24
if addr == "" {
25
addr = ":0"
26
}
@@ -38,6 +38,14 @@ func serveHTTP(addr string, serv *portal.RelayServer, nodeID string, bootstraps
38
serveAsset(appMux, "/favicon.png", "favicon.png", "image/png")
39
serveAsset(appMux, "/favicon.svg", "favicon.svg", "image/svg+xml")
40
41
+ if noIndex {
42
+ appMux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
43
+ w.Header().Set("Content-Type", "text/plain")
44
+ w.WriteHeader(http.StatusOK)
45
+ w.Write([]byte("User-agent: *\nDisallow: /\n"))
46
+ })
47
+ }
48
+
49
// Portal app assets (JS, CSS, etc.) - served from /app/
50
appMux.HandleFunc("/app/", func(w http.ResponseWriter, r *http.Request) {
51
utils.SetCORSHeaders(w)