block more user-agents.

block.go

@@ -11,9 +11,18 @@ import (
 func agentBlock(next http.HandlerFunc) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ua := r.Header.Get("User-Agent")
-		for _, bua := range []string{"Amazonbot", "semrush", "Bytespider", "AhrefsBot"} {
+		for _, bua := range []string{
+			"Amazonbot",
+			"semrush",
+			"Bytespider",
+			"AhrefsBot",
+			"DataForSeoBot",
+			"Yandex",
+			"meta-externalagent",
+			"DotBot",
+		} {
 			if strings.Contains(ua, bua) {
-				log.Debug().Str("ua", ua).Msg("user-agent blocked")
+				// log.Debug().Str("ua", ua).Msg("user-agent blocked")
 				http.Error(w, "Forbidden", http.StatusForbidden)
 				return
 			}

render_robots.go

@@ -7,7 +7,23 @@ import (
 
 func renderRobots(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "max-age=3600")
-	fmt.Fprintf(w, `User-agent: *
+	fmt.Fprintf(w, `
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: meta-externalagent
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /
+
+User-agent: dotbot
+Disallow: /
+
+User-agent: *
 Allow: /
 
 Sitemap: https://%s/npubs-archive.xml