diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ff031d9..1ccba58c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,25 @@ -2.5.0 / +2.5.0 / 2020-06-04 ================== **General** * Use a session invalidation strategy inspired by Django. Newly generated user sessions will now include a HMAC of the user's password. When the user's password is changed by someone other than the user the previous HMACs will no longer be valid and the user will be logged out when they next attempt to perform an action. +* A user and team's place, and score are now cached and invalidated on score changes. **API** * Add `/api/v1/challenges?view=admin` to allow admin users to see all challenges regardless of their visibility state * Add `/api/v1/users?view=admin` to allow admin users to see all users regardless of their hidden/banned state * Add `/api/v1/teams?view=admin` to allow admin users to see all teams regardless of their hidden/banned state +* The scoreboard endpoint `/api/v1/scoreboard` is now significantly more performant (20x) due to better response generation +* The top scoreboard endpoint `/api/v1/scoreboard/top/` is now more performant (3x) due to better response generation +* The scoreboard endpoint `/api/v1/scoreboard` will no longer show hidden/banned users in a non-hidden team **Deployment** * `docker-compose` now provides a basic nginx configuration and deploys nginx on port 80 +* `Dockerfile` now installs `python3` and `python3-dev` instead of `python` and `python-dev` because Alpine no longer provides those dependencies **Miscellaneous** * The `get_config` and `get_page` config utilities now use SQLAlchemy Core instead of SQLAlchemy ORM for slight speedups +* The `get_team_standings` and `get_user_standings` functions now return more data (id, oauth_id, name, score for regular users and banned, hidden as well for admins) * Update Flask-Migrate to 2.5.3 and regenerate the migration environment. Fixes using `%` signs in database passwords. diff --git a/CTFd/api/v1/scoreboard.py b/CTFd/api/v1/scoreboard.py index 5d44952a..b0f9ce2e 100644 --- a/CTFd/api/v1/scoreboard.py +++ b/CTFd/api/v1/scoreboard.py @@ -1,4 +1,7 @@ +from collections import defaultdict + from flask_restx import Namespace, Resource +from sqlalchemy.orm import joinedload from CTFd.cache import cache, make_cache_key from CTFd.models import Awards, Solves, Teams @@ -9,7 +12,7 @@ from CTFd.utils.decorators.visibility import ( check_score_visibility, ) from CTFd.utils.modes import TEAMS_MODE, generate_account_url, get_mode_as_word -from CTFd.utils.scores import get_standings +from CTFd.utils.scores import get_standings, get_user_standings scoreboard_namespace = Namespace( "scoreboard", description="Endpoint to retrieve scores" @@ -31,9 +34,23 @@ class ScoreboardList(Resource): team_ids = [] for team in standings: team_ids.append(team.account_id) - teams = Teams.query.filter(Teams.id.in_(team_ids)).all() + + # Get team objects with members explicitly loaded in + teams = ( + Teams.query.options(joinedload(Teams.members)) + .filter(Teams.id.in_(team_ids)) + .all() + ) + + # Sort according to team_ids order teams = [next(t for t in teams if t.id == id) for id in team_ids] + # Get user_standings as a dict so that we can more quickly get member scores + user_standings = get_user_standings() + users = {} + for u in user_standings: + users[u.user_id] = u + for i, x in enumerate(standings): entry = { "pos": i + 1, @@ -47,15 +64,30 @@ class ScoreboardList(Resource): if mode == TEAMS_MODE: members = [] + + # This code looks like it would be slow + # but it is faster than accessing each member's score individually for member in teams[i].members: - members.append( - { - "id": member.id, - "oauth_id": member.oauth_id, - "name": member.name, - "score": int(member.score), - } - ) + user = users.get(member.id) + if user: + members.append( + { + "id": user.user_id, + "oauth_id": user.oauth_id, + "name": user.name, + "score": int(user.score), + } + ) + else: + if member.hidden is False and member.banned is False: + members.append( + { + "id": member.id, + "oauth_id": member.oauth_id, + "name": member.name, + "score": 0, + } + ) entry["members"] = members @@ -88,38 +120,42 @@ class ScoreboardDetail(Resource): solves = solves.all() awards = awards.all() + # Build a mapping of accounts to their solves and awards + solves_mapper = defaultdict(list) + for solve in solves: + solves_mapper[solve.account_id].append( + { + "challenge_id": solve.challenge_id, + "account_id": solve.account_id, + "team_id": solve.team_id, + "user_id": solve.user_id, + "value": solve.challenge.value, + "date": isoformat(solve.date), + } + ) + + for award in awards: + solves_mapper[award.account_id].append( + { + "challenge_id": None, + "account_id": award.account_id, + "team_id": award.team_id, + "user_id": award.user_id, + "value": award.value, + "date": isoformat(award.date), + } + ) + + # Sort all solves by date + for team_id in solves_mapper: + solves_mapper[team_id] = sorted( + solves_mapper[team_id], key=lambda k: k["date"] + ) + for i, team in enumerate(team_ids): response[i + 1] = { "id": standings[i].account_id, "name": standings[i].name, - "solves": [], + "solves": solves_mapper.get(standings[i].account_id, []), } - for solve in solves: - if solve.account_id == team: - response[i + 1]["solves"].append( - { - "challenge_id": solve.challenge_id, - "account_id": solve.account_id, - "team_id": solve.team_id, - "user_id": solve.user_id, - "value": solve.challenge.value, - "date": isoformat(solve.date), - } - ) - for award in awards: - if award.account_id == team: - response[i + 1]["solves"].append( - { - "challenge_id": None, - "account_id": award.account_id, - "team_id": award.team_id, - "user_id": award.user_id, - "value": award.value, - "date": isoformat(award.date), - } - ) - response[i + 1]["solves"] = sorted( - response[i + 1]["solves"], key=lambda k: k["date"] - ) - return {"success": True, "data": response} diff --git a/CTFd/cache/__init__.py b/CTFd/cache/__init__.py index 9ea10e03..a06d003f 100644 --- a/CTFd/cache/__init__.py +++ b/CTFd/cache/__init__.py @@ -26,6 +26,7 @@ def clear_config(): def clear_standings(): + from CTFd.models import Users, Teams from CTFd.utils.scores import get_standings, get_team_standings, get_user_standings from CTFd.api.v1.scoreboard import ScoreboardDetail, ScoreboardList from CTFd.api import api @@ -33,6 +34,10 @@ def clear_standings(): cache.delete_memoized(get_standings) cache.delete_memoized(get_team_standings) cache.delete_memoized(get_user_standings) + cache.delete_memoized(Users.get_score) + cache.delete_memoized(Users.get_place) + cache.delete_memoized(Teams.get_score) + cache.delete_memoized(Teams.get_place) cache.delete(make_cache_key(path="scoreboard.listing")) cache.delete(make_cache_key(path=api.name + "." + ScoreboardList.endpoint)) cache.delete(make_cache_key(path=api.name + "." + ScoreboardDetail.endpoint)) diff --git a/CTFd/models/__init__.py b/CTFd/models/__init__.py index 629c2512..89c2a036 100644 --- a/CTFd/models/__init__.py +++ b/CTFd/models/__init__.py @@ -5,6 +5,8 @@ from flask_sqlalchemy import SQLAlchemy from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import column_property, validates +from CTFd.cache import cache + db = SQLAlchemy() ma = Marshmallow() @@ -331,6 +333,7 @@ class Users(db.Model): awards = awards.filter(Awards.date < dt) return awards.all() + @cache.memoize() def get_score(self, admin=False): score = db.func.sum(Challenges.value).label("score") user = ( @@ -363,6 +366,7 @@ class Users(db.Model): else: return 0 + @cache.memoize() def get_place(self, admin=False, numeric=False): """ This method is generally a clone of CTFd.scoreboard.get_standings. @@ -375,12 +379,13 @@ class Users(db.Model): standings = get_user_standings(admin=admin) - try: - n = standings.index((self.id,)) + 1 - if numeric: - return n - return ordinalize(n) - except ValueError: + for i, user in enumerate(standings): + if user.user_id == self.id: + n = i + 1 + if numeric: + return n + return ordinalize(n) + else: return None @@ -401,7 +406,9 @@ class Teams(db.Model): password = db.Column(db.String(128)) secret = db.Column(db.String(128)) - members = db.relationship("Users", backref="team", foreign_keys="Users.team_id") + members = db.relationship( + "Users", backref="team", foreign_keys="Users.team_id", lazy="joined" + ) # Supplementary attributes website = db.Column(db.String(128)) @@ -499,12 +506,14 @@ class Teams(db.Model): return awards.all() + @cache.memoize() def get_score(self, admin=False): score = 0 for member in self.members: score += member.get_score(admin=admin) return score + @cache.memoize() def get_place(self, admin=False, numeric=False): """ This method is generally a clone of CTFd.scoreboard.get_standings. @@ -517,12 +526,13 @@ class Teams(db.Model): standings = get_team_standings(admin=admin) - try: - n = standings.index((self.id,)) + 1 - if numeric: - return n - return ordinalize(n) - except ValueError: + for i, team in enumerate(standings): + if team.team_id == self.id: + n = i + 1 + if numeric: + return n + return ordinalize(n) + else: return None diff --git a/CTFd/themes/admin/assets/js/pages/configs.js b/CTFd/themes/admin/assets/js/pages/configs.js index 7b93dba1..91cbc469 100644 --- a/CTFd/themes/admin/assets/js/pages/configs.js +++ b/CTFd/themes/admin/assets/js/pages/configs.js @@ -231,19 +231,25 @@ function insertTimezones(target) { } $(() => { - CodeMirror.fromTextArea(document.getElementById("theme-header"), { - lineNumbers: true, - lineWrapping: true, - mode: "htmlmixed", - htmlMode: true - }); + const theme_header_editor = CodeMirror.fromTextArea( + document.getElementById("theme-header"), + { + lineNumbers: true, + lineWrapping: true, + mode: "htmlmixed", + htmlMode: true + } + ); - CodeMirror.fromTextArea(document.getElementById("theme-footer"), { - lineNumbers: true, - lineWrapping: true, - mode: "htmlmixed", - htmlMode: true - }); + const theme_footer_editor = CodeMirror.fromTextArea( + document.getElementById("theme-footer"), + { + lineNumbers: true, + lineWrapping: true, + mode: "htmlmixed", + htmlMode: true + } + ); insertTimezones($("#start-timezone")); insertTimezones($("#end-timezone")); @@ -256,7 +262,7 @@ $(() => { $("#import-button").click(importConfig); $("#config-color-update").click(function() { const hex_code = $("#config-color-picker").val(); - const user_css = $("#theme-header").val(); + const user_css = theme_header_editor.getValue(); let new_css; if (user_css.length) { let css_vars = `theme-color: ${hex_code};`; @@ -269,7 +275,7 @@ $(() => { `.jumbotron{background-color: var(--theme-color) !important;}\n` + `\n`; } - $("#theme-header").val(new_css); + theme_header_editor.getDoc().setValue(new_css); }); $(".start-date").change(function() { diff --git a/CTFd/themes/admin/static/js/pages/configs.dev.js b/CTFd/themes/admin/static/js/pages/configs.dev.js index ce5f6318..6aac5609 100644 --- a/CTFd/themes/admin/static/js/pages/configs.dev.js +++ b/CTFd/themes/admin/static/js/pages/configs.dev.js @@ -162,7 +162,7 @@ /***/ (function(module, exports, __webpack_require__) { ; -eval("\n\n__webpack_require__(/*! ./main */ \"./CTFd/themes/admin/assets/js/pages/main.js\");\n\n__webpack_require__(/*! core/utils */ \"./CTFd/themes/core/assets/js/utils.js\");\n\n__webpack_require__(/*! bootstrap/js/dist/tab */ \"./node_modules/bootstrap/js/dist/tab.js\");\n\nvar _momentTimezone = _interopRequireDefault(__webpack_require__(/*! moment-timezone */ \"./node_modules/moment-timezone/index.js\"));\n\nvar _CTFd = _interopRequireDefault(__webpack_require__(/*! core/CTFd */ \"./CTFd/themes/core/assets/js/CTFd.js\"));\n\nvar _helpers = _interopRequireDefault(__webpack_require__(/*! core/helpers */ \"./CTFd/themes/core/assets/js/helpers.js\"));\n\nvar _jquery = _interopRequireDefault(__webpack_require__(/*! jquery */ \"./node_modules/jquery/dist/jquery.js\"));\n\nvar _ezq = __webpack_require__(/*! core/ezq */ \"./CTFd/themes/core/assets/js/ezq.js\");\n\nvar _codemirror = _interopRequireDefault(__webpack_require__(/*! codemirror */ \"./node_modules/codemirror/lib/codemirror.js\"));\n\n__webpack_require__(/*! codemirror/mode/htmlmixed/htmlmixed.js */ \"./node_modules/codemirror/mode/htmlmixed/htmlmixed.js\");\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }\n\nfunction loadTimestamp(place, timestamp) {\n if (typeof timestamp == \"string\") {\n timestamp = parseInt(timestamp, 10);\n }\n\n var m = (0, _momentTimezone.default)(timestamp * 1000);\n (0, _jquery.default)(\"#\" + place + \"-month\").val(m.month() + 1); // Months are zero indexed (http://momentjs.com/docs/#/get-set/month/)\n\n (0, _jquery.default)(\"#\" + place + \"-day\").val(m.date());\n (0, _jquery.default)(\"#\" + place + \"-year\").val(m.year());\n (0, _jquery.default)(\"#\" + place + \"-hour\").val(m.hour());\n (0, _jquery.default)(\"#\" + place + \"-minute\").val(m.minute());\n loadDateValues(place);\n}\n\nfunction loadDateValues(place) {\n var month = (0, _jquery.default)(\"#\" + place + \"-month\").val();\n var day = (0, _jquery.default)(\"#\" + place + \"-day\").val();\n var year = (0, _jquery.default)(\"#\" + place + \"-year\").val();\n var hour = (0, _jquery.default)(\"#\" + place + \"-hour\").val();\n var minute = (0, _jquery.default)(\"#\" + place + \"-minute\").val();\n var timezone = (0, _jquery.default)(\"#\" + place + \"-timezone\").val();\n var utc = convertDateToMoment(month, day, year, hour, minute);\n\n if (isNaN(utc.unix())) {\n (0, _jquery.default)(\"#\" + place).val(\"\");\n (0, _jquery.default)(\"#\" + place + \"-local\").val(\"\");\n (0, _jquery.default)(\"#\" + place + \"-zonetime\").val(\"\");\n } else {\n (0, _jquery.default)(\"#\" + place).val(utc.unix());\n (0, _jquery.default)(\"#\" + place + \"-local\").val(utc.local().format(\"dddd, MMMM Do YYYY, h:mm:ss a zz\"));\n (0, _jquery.default)(\"#\" + place + \"-zonetime\").val(utc.tz(timezone).format(\"dddd, MMMM Do YYYY, h:mm:ss a zz\"));\n }\n}\n\nfunction convertDateToMoment(month, day, year, hour, minute) {\n var month_num = month.toString();\n\n if (month_num.length == 1) {\n month_num = \"0\" + month_num;\n }\n\n var day_str = day.toString();\n\n if (day_str.length == 1) {\n day_str = \"0\" + day_str;\n }\n\n var hour_str = hour.toString();\n\n if (hour_str.length == 1) {\n hour_str = \"0\" + hour_str;\n }\n\n var min_str = minute.toString();\n\n if (min_str.length == 1) {\n min_str = \"0\" + min_str;\n } // 2013-02-08 24:00\n\n\n var date_string = year.toString() + \"-\" + month_num + \"-\" + day_str + \" \" + hour_str + \":\" + min_str + \":00\";\n return (0, _momentTimezone.default)(date_string, _momentTimezone.default.ISO_8601);\n}\n\nfunction updateConfigs(event) {\n event.preventDefault();\n var obj = (0, _jquery.default)(this).serializeJSON();\n var params = {};\n\n if (obj.mail_useauth === false) {\n obj.mail_username = null;\n obj.mail_password = null;\n } else {\n if (obj.mail_username === \"\") {\n delete obj.mail_username;\n }\n\n if (obj.mail_password === \"\") {\n delete obj.mail_password;\n }\n }\n\n Object.keys(obj).forEach(function (x) {\n if (obj[x] === \"true\") {\n params[x] = true;\n } else if (obj[x] === \"false\") {\n params[x] = false;\n } else {\n params[x] = obj[x];\n }\n });\n\n _CTFd.default.api.patch_config_list({}, params).then(function (response) {\n window.location.reload();\n });\n}\n\nfunction uploadLogo(event) {\n event.preventDefault();\n var form = event.target;\n\n _helpers.default.files.upload(form, {}, function (response) {\n var f = response.data[0];\n var params = {\n value: f.location\n };\n\n _CTFd.default.fetch(\"/api/v1/configs/ctf_logo\", {\n method: \"PATCH\",\n body: JSON.stringify(params)\n }).then(function (response) {\n return response.json();\n }).then(function (response) {\n if (response.success) {\n window.location.reload();\n } else {\n ezAlert({\n title: \"Error!\",\n body: \"Logo uploading failed!\",\n button: \"Okay\"\n });\n }\n });\n });\n}\n\nfunction removeLogo() {\n (0, _ezq.ezQuery)({\n title: \"Remove logo\",\n body: \"Are you sure you'd like to remove the CTF logo?\",\n success: function success() {\n var params = {\n value: null\n };\n\n _CTFd.default.api.patch_config({\n configKey: \"ctf_logo\"\n }, params).then(function (response) {\n window.location.reload();\n });\n }\n });\n}\n\nfunction importConfig(event) {\n event.preventDefault();\n var import_file = document.getElementById(\"import-file\").files[0];\n var form_data = new FormData();\n form_data.append(\"backup\", import_file);\n form_data.append(\"nonce\", _CTFd.default.config.csrfNonce);\n var pg = (0, _ezq.ezProgressBar)({\n width: 0,\n title: \"Upload Progress\"\n });\n\n _jquery.default.ajax({\n url: _CTFd.default.config.urlRoot + \"/admin/import\",\n type: \"POST\",\n data: form_data,\n processData: false,\n contentType: false,\n statusCode: {\n 500: function _(resp) {\n console.log(resp.responseText);\n alert(resp.responseText);\n }\n },\n xhr: function xhr() {\n var xhr = _jquery.default.ajaxSettings.xhr();\n\n xhr.upload.onprogress = function (e) {\n if (e.lengthComputable) {\n var width = e.loaded / e.total * 100;\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: width\n });\n }\n };\n\n return xhr;\n },\n success: function success(data) {\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: 100\n });\n setTimeout(function () {\n pg.modal(\"hide\");\n }, 500);\n setTimeout(function () {\n window.location.reload();\n }, 700);\n }\n });\n}\n\nfunction exportConfig(event) {\n event.preventDefault();\n var href = _CTFd.default.config.urlRoot + \"/admin/export\";\n window.location.href = (0, _jquery.default)(this).attr(\"href\");\n}\n\nfunction insertTimezones(target) {\n var current = (0, _jquery.default)(\"