block more user-agents.

render_robots.go

@@ -7,7 +7,23 @@ import (
 
 func renderRobots(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "max-age=3600")
-	fmt.Fprintf(w, `User-agent: *
+	fmt.Fprintf(w, `
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: meta-externalagent
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /
+
+User-agent: dotbot
+Disallow: /
+
+User-agent: *
 Allow: /
 
 Sitemap: https://%s/npubs-archive.xml