Merge remote-tracking branch 'origin/master' into 3.0.0-dev

CHANGELOG.md

@@ -1,19 +1,25 @@
-2.5.0 /
+2.5.0 / 2020-06-04
 ==================
 
 **General**
 * Use a session invalidation strategy inspired by Django. Newly generated user sessions will now include a HMAC of the user's password. When the user's password is changed by someone other than the user the previous HMACs will no longer be valid and the user will be logged out when they next attempt to perform an action.
+* A user and team's place, and score are now cached and invalidated on score changes.
 
 **API**
 * Add `/api/v1/challenges?view=admin` to allow admin users to see all challenges regardless of their visibility state
 * Add `/api/v1/users?view=admin` to allow admin users to see all users regardless of their hidden/banned state
 * Add `/api/v1/teams?view=admin` to allow admin users to see all teams regardless of their hidden/banned state
+* The scoreboard endpoint `/api/v1/scoreboard` is now significantly more performant (20x) due to better response generation
+* The top scoreboard endpoint `/api/v1/scoreboard/top/<count>` is now more performant (3x) due to better response generation
+* The scoreboard endpoint `/api/v1/scoreboard` will no longer show hidden/banned users in a non-hidden team
 
 **Deployment**
 * `docker-compose` now provides a basic nginx configuration and deploys nginx on port 80
+* `Dockerfile` now installs `python3` and `python3-dev` instead of `python` and `python-dev` because Alpine no longer provides those dependencies
 
 **Miscellaneous**
 * The `get_config` and `get_page` config utilities now use SQLAlchemy Core instead of SQLAlchemy ORM for slight speedups
+* The `get_team_standings` and `get_user_standings` functions now return more data (id, oauth_id, name, score for regular users and banned, hidden as well for admins)
 * Update Flask-Migrate to 2.5.3 and regenerate the migration environment. Fixes using `%` signs in database passwords.
 
 

CTFd/api/v1/scoreboard.py

@@ -1,4 +1,7 @@
+from collections import defaultdict
+
 from flask_restx import Namespace, Resource
+from sqlalchemy.orm import joinedload
 
 from CTFd.cache import cache, make_cache_key
 from CTFd.models import Awards, Solves, Teams
@@ -9,7 +12,7 @@ from CTFd.utils.decorators.visibility import (
     check_score_visibility,
 )
 from CTFd.utils.modes import TEAMS_MODE, generate_account_url, get_mode_as_word
-from CTFd.utils.scores import get_standings
+from CTFd.utils.scores import get_standings, get_user_standings
 
 scoreboard_namespace = Namespace(
     "scoreboard", description="Endpoint to retrieve scores"
@@ -31,9 +34,23 @@ class ScoreboardList(Resource):
             team_ids = []
             for team in standings:
                 team_ids.append(team.account_id)
-            teams = Teams.query.filter(Teams.id.in_(team_ids)).all()
+
+            # Get team objects with members explicitly loaded in
+            teams = (
+                Teams.query.options(joinedload(Teams.members))
+                .filter(Teams.id.in_(team_ids))
+                .all()
+            )
+
+            # Sort according to team_ids order
             teams = [next(t for t in teams if t.id == id) for id in team_ids]
 
+            # Get user_standings as a dict so that we can more quickly get member scores
+            user_standings = get_user_standings()
+            users = {}
+            for u in user_standings:
+                users[u.user_id] = u
+
         for i, x in enumerate(standings):
             entry = {
                 "pos": i + 1,
@@ -47,15 +64,30 @@ class ScoreboardList(Resource):
 
             if mode == TEAMS_MODE:
                 members = []
+
+                # This code looks like it would be slow
+                # but it is faster than accessing each member's score individually
                 for member in teams[i].members:
-                    members.append(
-                        {
-                            "id": member.id,
-                            "oauth_id": member.oauth_id,
-                            "name": member.name,
-                            "score": int(member.score),
-                        }
-                    )
+                    user = users.get(member.id)
+                    if user:
+                        members.append(
+                            {
+                                "id": user.user_id,
+                                "oauth_id": user.oauth_id,
+                                "name": user.name,
+                                "score": int(user.score),
+                            }
+                        )
+                    else:
+                        if member.hidden is False and member.banned is False:
+                            members.append(
+                                {
+                                    "id": member.id,
+                                    "oauth_id": member.oauth_id,
+                                    "name": member.name,
+                                    "score": 0,
+                                }
+                            )
 
                 entry["members"] = members
 
@@ -88,38 +120,42 @@ class ScoreboardDetail(Resource):
         solves = solves.all()
         awards = awards.all()
 
+        # Build a mapping of accounts to their solves and awards
+        solves_mapper = defaultdict(list)
+        for solve in solves:
+            solves_mapper[solve.account_id].append(
+                {
+                    "challenge_id": solve.challenge_id,
+                    "account_id": solve.account_id,
+                    "team_id": solve.team_id,
+                    "user_id": solve.user_id,
+                    "value": solve.challenge.value,
+                    "date": isoformat(solve.date),
+                }
+            )
+
+        for award in awards:
+            solves_mapper[award.account_id].append(
+                {
+                    "challenge_id": None,
+                    "account_id": award.account_id,
+                    "team_id": award.team_id,
+                    "user_id": award.user_id,
+                    "value": award.value,
+                    "date": isoformat(award.date),
+                }
+            )
+
+        # Sort all solves by date
+        for team_id in solves_mapper:
+            solves_mapper[team_id] = sorted(
+                solves_mapper[team_id], key=lambda k: k["date"]
+            )
+
         for i, team in enumerate(team_ids):
             response[i + 1] = {
                 "id": standings[i].account_id,
                 "name": standings[i].name,
-                "solves": [],
+                "solves": solves_mapper.get(standings[i].account_id, []),
             }
-            for solve in solves:
-                if solve.account_id == team:
-                    response[i + 1]["solves"].append(
-                        {
-                            "challenge_id": solve.challenge_id,
-                            "account_id": solve.account_id,
-                            "team_id": solve.team_id,
-                            "user_id": solve.user_id,
-                            "value": solve.challenge.value,
-                            "date": isoformat(solve.date),
-                        }
-                    )
-            for award in awards:
-                if award.account_id == team:
-                    response[i + 1]["solves"].append(
-                        {
-                            "challenge_id": None,
-                            "account_id": award.account_id,
-                            "team_id": award.team_id,
-                            "user_id": award.user_id,
-                            "value": award.value,
-                            "date": isoformat(award.date),
-                        }
-                    )
-            response[i + 1]["solves"] = sorted(
-                response[i + 1]["solves"], key=lambda k: k["date"]
-            )
-
         return {"success": True, "data": response}

CTFd/cache/__init__.py

@@ -26,6 +26,7 @@ def clear_config():
 
 
 def clear_standings():
+    from CTFd.models import Users, Teams
     from CTFd.utils.scores import get_standings, get_team_standings, get_user_standings
     from CTFd.api.v1.scoreboard import ScoreboardDetail, ScoreboardList
     from CTFd.api import api
@@ -33,6 +34,10 @@ def clear_standings():
     cache.delete_memoized(get_standings)
     cache.delete_memoized(get_team_standings)
     cache.delete_memoized(get_user_standings)
+    cache.delete_memoized(Users.get_score)
+    cache.delete_memoized(Users.get_place)
+    cache.delete_memoized(Teams.get_score)
+    cache.delete_memoized(Teams.get_place)
     cache.delete(make_cache_key(path="scoreboard.listing"))
     cache.delete(make_cache_key(path=api.name + "." + ScoreboardList.endpoint))
     cache.delete(make_cache_key(path=api.name + "." + ScoreboardDetail.endpoint))

CTFd/models/__init__.py

@@ -5,6 +5,8 @@ from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import column_property, validates
 
+from CTFd.cache import cache
+
 db = SQLAlchemy()
 ma = Marshmallow()
 
@@ -331,6 +333,7 @@ class Users(db.Model):
             awards = awards.filter(Awards.date < dt)
         return awards.all()
 
+    @cache.memoize()
     def get_score(self, admin=False):
         score = db.func.sum(Challenges.value).label("score")
         user = (
@@ -363,6 +366,7 @@ class Users(db.Model):
         else:
             return 0
 
+    @cache.memoize()
     def get_place(self, admin=False, numeric=False):
         """
         This method is generally a clone of CTFd.scoreboard.get_standings.
@@ -375,12 +379,13 @@ class Users(db.Model):
 
         standings = get_user_standings(admin=admin)
 
-        try:
-            n = standings.index((self.id,)) + 1
-            if numeric:
-                return n
-            return ordinalize(n)
-        except ValueError:
+        for i, user in enumerate(standings):
+            if user.user_id == self.id:
+                n = i + 1
+                if numeric:
+                    return n
+                return ordinalize(n)
+        else:
             return None
 
 
@@ -401,7 +406,9 @@ class Teams(db.Model):
     password = db.Column(db.String(128))
     secret = db.Column(db.String(128))
 
-    members = db.relationship("Users", backref="team", foreign_keys="Users.team_id")
+    members = db.relationship(
+        "Users", backref="team", foreign_keys="Users.team_id", lazy="joined"
+    )
 
     # Supplementary attributes
     website = db.Column(db.String(128))
@@ -499,12 +506,14 @@ class Teams(db.Model):
 
         return awards.all()
 
+    @cache.memoize()
     def get_score(self, admin=False):
         score = 0
         for member in self.members:
             score += member.get_score(admin=admin)
         return score
 
+    @cache.memoize()
     def get_place(self, admin=False, numeric=False):
         """
         This method is generally a clone of CTFd.scoreboard.get_standings.
@@ -517,12 +526,13 @@ class Teams(db.Model):
 
         standings = get_team_standings(admin=admin)
 
-        try:
-            n = standings.index((self.id,)) + 1
-            if numeric:
-                return n
-            return ordinalize(n)
-        except ValueError:
+        for i, team in enumerate(standings):
+            if team.team_id == self.id:
+                n = i + 1
+                if numeric:
+                    return n
+                return ordinalize(n)
+        else:
             return None
 
 

CTFd/themes/admin/assets/js/pages/configs.js

@@ -231,19 +231,25 @@ function insertTimezones(target) {
 }
 
 $(() => {
-  CodeMirror.fromTextArea(document.getElementById("theme-header"), {
-    lineNumbers: true,
-    lineWrapping: true,
-    mode: "htmlmixed",
-    htmlMode: true
-  });
+  const theme_header_editor = CodeMirror.fromTextArea(
+    document.getElementById("theme-header"),
+    {
+      lineNumbers: true,
+      lineWrapping: true,
+      mode: "htmlmixed",
+      htmlMode: true
+    }
+  );
 
-  CodeMirror.fromTextArea(document.getElementById("theme-footer"), {
-    lineNumbers: true,
-    lineWrapping: true,
-    mode: "htmlmixed",
-    htmlMode: true
-  });
+  const theme_footer_editor = CodeMirror.fromTextArea(
+    document.getElementById("theme-footer"),
+    {
+      lineNumbers: true,
+      lineWrapping: true,
+      mode: "htmlmixed",
+      htmlMode: true
+    }
+  );
 
   insertTimezones($("#start-timezone"));
   insertTimezones($("#end-timezone"));
@@ -256,7 +262,7 @@ $(() => {
   $("#import-button").click(importConfig);
   $("#config-color-update").click(function() {
     const hex_code = $("#config-color-picker").val();
-    const user_css = $("#theme-header").val();
+    const user_css = theme_header_editor.getValue();
     let new_css;
     if (user_css.length) {
       let css_vars = `theme-color: ${hex_code};`;
@@ -269,7 +275,7 @@ $(() => {
         `.jumbotron{background-color: var(--theme-color) !important;}\n` +
         `</style>\n`;
     }
-    $("#theme-header").val(new_css);
+    theme_header_editor.getDoc().setValue(new_css);
   });
 
   $(".start-date").change(function() {

CTFd/utils/scores/__init__.py

@@ -159,13 +159,25 @@ def get_team_standings(count=None, admin=False):
 
     if admin:
         standings_query = (
-            db.session.query(Teams.id.label("team_id"))
+            db.session.query(
+                Teams.id.label("team_id"),
+                Teams.oauth_id.label("oauth_id"),
+                Teams.name.label("name"),
+                Teams.hidden,
+                Teams.banned,
+                sumscores.columns.score,
+            )
             .join(sumscores, Teams.id == sumscores.columns.team_id)
             .order_by(sumscores.columns.score.desc(), sumscores.columns.id)
         )
     else:
         standings_query = (
-            db.session.query(Teams.id.label("team_id"))
+            db.session.query(
+                Teams.id.label("team_id"),
+                Teams.oauth_id.label("oauth_id"),
+                Teams.name.label("name"),
+                sumscores.columns.score,
+            )
             .join(sumscores, Teams.id == sumscores.columns.team_id)
             .filter(Teams.banned == False)
             .filter(Teams.hidden == False)
@@ -225,13 +237,25 @@ def get_user_standings(count=None, admin=False):
 
     if admin:
         standings_query = (
-            db.session.query(Users.id.label("user_id"))
+            db.session.query(
+                Users.id.label("user_id"),
+                Users.oauth_id.label("oauth_id"),
+                Users.name.label("name"),
+                Users.hidden,
+                Users.banned,
+                sumscores.columns.score,
+            )
             .join(sumscores, Users.id == sumscores.columns.user_id)
             .order_by(sumscores.columns.score.desc(), sumscores.columns.id)
         )
     else:
         standings_query = (
-            db.session.query(Users.id.label("user_id"))
+            db.session.query(
+                Users.id.label("user_id"),
+                Users.oauth_id.label("oauth_id"),
+                Users.name.label("name"),
+                sumscores.columns.score,
+            )
             .join(sumscores, Users.id == sumscores.columns.user_id)
             .filter(Users.banned == False, Users.hidden == False)
             .order_by(sumscores.columns.score.desc(), sumscores.columns.id)

docs/conf.py

@@ -26,7 +26,7 @@ author = u"Kevin Chung"
 # The short X.Y version
 version = u""
 # The full version, including alpha/beta/rc tags
-release = u"2.4.3"
+release = u"2.5.0"
 
 
 # -- General configuration ---------------------------------------------------

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ctfd",
-  "version": "2.4.3",
+  "version": "2.5.0",
   "description": "CTFd is a Capture The Flag framework focusing on ease of use and customizability. It comes with everything you need to run a CTF and it's easy to customize with plugins and themes.",
   "main": "index.js",
   "directories": {