Add support for robots.txt (#2269)

* Adds support for admins to control `robots.txt`
* Closes #2141

CTFd/themes/admin/templates/config.html

@@ -23,6 +23,9 @@
 				<li class="nav-item">
 					<a class="nav-link rounded-0" href="#accounts" role="tab" data-toggle="tab">Accounts</a>
 				</li>
+				<li class="nav-item">
+					<a class="nav-link rounded-0" href="#pages" role="tab" data-toggle="tab">Pages</a>
+				</li>
 				<li class="nav-item">
 					<a class="nav-link rounded-0" href="#fields" role="tab" data-toggle="tab">Custom Fields</a>
 				</li>
@@ -73,6 +76,8 @@
 
 				{% include "admin/configs/accounts.html" %}
 
+				{% include "admin/configs/pages.html" %}
+
 				{% include "admin/configs/fields.html" %}
 
 				{% include "admin/configs/mlc.html" %}

CTFd/themes/admin/templates/configs/pages.html

@@ -0,0 +1,14 @@
+<div role="tabpanel" class="tab-pane config-section" id="pages">
+	<form  method="POST" enctype="multipart/form-data">
+		<div class="form-group">
+			<label>
+				Robots.txt
+				<small class="form-text text-muted">
+					The robots.txt file contains instructions that suggest to bots which webpages they can and cannot access
+				</small>
+			</label>
+			<textarea class="form-control" id="theme-footer" name="robots_txt" rows="7">{{ robots_txt or "User-agent: *\nDisallow: /admin\n" }}</textarea>
+		</div>
+		<button type="submit" class="btn btn-md btn-primary float-right">Update</button>
+	</form>
+</div>

CTFd/utils/initialization/__init__.py

@@ -204,6 +204,7 @@ def init_request_processors(app):
                 "views.themes",
                 "views.files",
                 "views.healthcheck",
+                "views.robots",
             ):
                 return
             else:

CTFd/views.py

@@ -2,7 +2,15 @@ import os
 
 from flask import Blueprint, abort
 from flask import current_app as app
-from flask import redirect, render_template, request, send_file, session, url_for
+from flask import (
+    make_response,
+    redirect,
+    render_template,
+    request,
+    send_file,
+    session,
+    url_for,
+)
 from flask.helpers import safe_join
 from jinja2.exceptions import TemplateNotFound
 from sqlalchemy.exc import IntegrityError
@@ -514,3 +522,11 @@ def healthcheck():
     if check_config() is False:
         return "ERR", 500
     return "OK", 200
+
+
+@views.route("/robots.txt")
+def robots():
+    text = get_config("robots_txt", "User-agent: *\nDisallow: /admin\n")
+    r = make_response(text, 200)
+    r.mimetype = "text/plain"
+    return r

tests/test_views.py

@@ -451,3 +451,19 @@ def test_user_can_access_files_if_view_after_ctf():
             rmdir(directory)
 
     destroy_ctfd(app)
+
+
+def test_robots_txt():
+    """Does the robots.txt page work"""
+    app = create_ctfd()
+    with app.app_context():
+        with app.test_client() as client:
+            r = client.get("/robots.txt")
+            assert r.status_code == 200
+            assert r.get_data(as_text=True) == "User-agent: *\nDisallow: /admin\n"
+        set_config("robots_txt", "testing")
+        with app.test_client() as client:
+            r = client.get("/robots.txt")
+            assert r.status_code == 200
+            assert r.get_data(as_text=True) == "testing"
+    destroy_ctfd(app)