From 4fde0368db80aa5c779cc9bcb74ef67b4f5f39d6 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 22 Nov 2018 11:05:47 -0500 Subject: [PATCH] 2.0.0 Supplementary Changes (#744) * Clean up models a little * Add 1.2.0 migration script * Add 2.0.0 CHANGELOG * Fix S3 uploader * Update config.py to grab S3 settings from envvars --- CHANGELOG.md | 69 ++++ CTFd/config.py | 8 +- CTFd/models/__init__.py | 4 +- CTFd/utils/uploads/uploaders.py | 2 +- migrations/1_2_0_upgrade_2_0_0.py | 522 ++++++++++++++++++++++++++++++ 5 files changed, 598 insertions(+), 7 deletions(-) create mode 100644 migrations/1_2_0_upgrade_2_0_0.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b2c47ba..350c5a9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,72 @@ +2.0.0 / 2018-11-22 +================== + +2.0.0 is a significant change. +If you are upgrading from a prior version be sure to make backups before upgrading. + +In addition, 2.0.0 is a backwards-incompatible release. Many plugins will be broken in CTFd 2.0.0, and if you're having +trouble updating your plugins please join [the CTFd Slack](https://slack.ctfd.io/) for help and discussion. + +**General** + +* Seperation of Teams into Users and Teams. +* Integration with MajorLeagueCyber. (https://majorleaguecyber.org) +* Data is now provided to the front-end via the REST API. (#551) + * Javascript uses `fetch()` to consume the REST API. +* Dynamic Challenges built in. +* S3 Uploader built in. (#661) +* Real time notifications. (#600) +* Email address domain whitelisting. (#603) +* Database exporting to CSV. (#656) +* Imports/Exports rewritten to act as backups. + * Importing no longer stacks values. + * Exports are no longer partial. +* Reset CTF from config panel (Remove all users, solves, fails. i.e. only keep Challenge data.) (#639) +* Countries are pre-determined and selectable instead of being user-entered. + * Countries stored based on country code. + * Based on https://github.com/umpirsky/country-list/blob/master/data/en_US/country.csv. +* Sessions are no longer stored using secure cookies. (#658) + * Sessions are now stored server side in a cache (`filesystem` or `redis`) allowing for session revocation. +* Challenges can now have requirements which must be met before the challenge can be seen/solved. +* Workshop mode, score hiding, registration hiding, challenge hiding have been changed to visibility settings. +* Users and Teams can now be banned preventing access to the CTF. +* Dockerfile improvements. + * WORKERS count in `docker-entrypoint.sh` defaults to 1. (#716) + * `docker-entrypoint.sh` exits on any error. (#717) +* Increased test coverage. +* Migrations have been reset. + * If upgrading from 1.2.0: + 1. Make all necessary backups. Backup the database, uploads folder, and source code directory. + 2. Upgrade the source code directory. + 3. Set the `DATABASE_URL` in `CTFd/config.py`. + 3. Run the upgrade script from the CTFd folder i.e. `python migrations/1_2_0_upgrade_2_0_0.py`. + 4. Setup the rest of CTFd and run normally. + +**Themes** + +* Data is now provided to the front-end via the REST API. + * Javascript uses `fetch()` to consume the REST API. +* The admin theme is no longer considered seperated from the core theme and should always be together. +* Themes now use `url_for()` to generate URLs instead of hardcoding. +* socket.io is used to connect to CTFd to receive notifications. +* `ctf_name()` renamed to `get_ctf_name()` in themes. +* `ctf_logo()` renamed to `get_ctf_logo()` in themes. +* `ctf_theme()` renamed to `get_ctf_theme()` in themes. +* Update Font-Awesome to 5.4.1. +* Update moment.js to 2.22.2. (#704) + +**Plugins** + +* Plugins are loaded in `sorted()` order +* Rename challenge type plugins to use .html and have simplified names. (create, update, view) +* Many functions moved around because utils.py has been broken up and refactored. (#475) +* Marshmallow (https://marshmallow.readthedocs.io) is now used by the REST API to validate and serialize/deserialize data. + * Marshmallow schemas and views are used to restrict SQLAlchemy columns to user types. +* The REST API features swagger support but this requires more utilization internally. +* Email registration regex relaxed. (#693) +* Many functions have moved and now have dedicated utils packages for their category. + + 1.2.0 / 2018-05-04 ================== diff --git a/CTFd/config.py b/CTFd/config.py index 6bc2949f..e688e548 100644 --- a/CTFd/config.py +++ b/CTFd/config.py @@ -165,10 +165,10 @@ class Config(object): UPLOAD_FOLDER = os.environ.get('UPLOAD_FOLDER') or \ os.path.join(os.path.dirname(os.path.abspath(__file__)), 'uploads') elif UPLOAD_PROVIDER == 's3': - AWS_ACCESS_KEY_ID = None - AWS_SECRET_ACCESS_KEY = None - AWS_S3_BUCKET = None - AWS_S3_ENDPOINT_URL = None + AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') or '' + AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') or '' + AWS_S3_BUCKET = os.environ.get('AWS_S3_BUCKET') or '' + AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL') or '' ''' === OPTIONAL === diff --git a/CTFd/models/__init__.py b/CTFd/models/__init__.py index 490e7dde..0ff23e59 100644 --- a/CTFd/models/__init__.py +++ b/CTFd/models/__init__.py @@ -729,7 +729,7 @@ class Unlocks(db.Model): return self.user_id def __repr__(self): - return '' % self.teamid + return '' % self.id class HintUnlocks(Unlocks): @@ -756,7 +756,7 @@ class Tracking(db.Model): super(Tracking, self).__init__(**kwargs) def __repr__(self): - return '' % self.team + return '' % self.ip class Configs(db.Model): diff --git a/CTFd/utils/uploads/uploaders.py b/CTFd/utils/uploads/uploaders.py index 2089cf5e..06a5d1c5 100644 --- a/CTFd/utils/uploads/uploaders.py +++ b/CTFd/utils/uploads/uploaders.py @@ -77,7 +77,7 @@ class S3Uploader(BaseUploader): ) return client - def _clean_filename(c): + def _clean_filename(self, c): if c in string.ascii_letters + string.digits + '-' + '_' + '.': return True diff --git a/migrations/1_2_0_upgrade_2_0_0.py b/migrations/1_2_0_upgrade_2_0_0.py new file mode 100644 index 00000000..cd09c9bc --- /dev/null +++ b/migrations/1_2_0_upgrade_2_0_0.py @@ -0,0 +1,522 @@ +import sys +import os + +# This is important to allow access to the CTFd application factory +sys.path.append(os.getcwd()) + +import datetime +import hashlib +import netaddr +from flask_sqlalchemy import SQLAlchemy +from passlib.hash import bcrypt_sha256 +from sqlalchemy.sql.expression import union_all +from CTFd import config, create_app +from sqlalchemy_utils import ( + drop_database, +) +from six.moves import input + +import dataset + + +def sha512(string): + return str(hashlib.sha512(string).hexdigest()) + + +def ip2long(ip): + '''Converts a user's IP address into an integer/long''' + return int(netaddr.IPAddress(ip)) + + +def long2ip(ip_int): + '''Converts a saved integer/long back into an IP address''' + return str(netaddr.IPAddress(ip_int)) + + +db = SQLAlchemy() + + +class Pages(db.Model): + id = db.Column(db.Integer, primary_key=True) + auth_required = db.Column(db.Boolean) + title = db.Column(db.String(80)) + route = db.Column(db.Text, unique=True) + html = db.Column(db.Text) + draft = db.Column(db.Boolean) + + def __init__(self, title, route, html, draft=True, auth_required=False): + self.title = title + self.route = route + self.html = html + self.draft = draft + self.auth_required = auth_required + + def __repr__(self): + return "".format(self.route) + + +class Challenges(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(80)) + description = db.Column(db.Text) + max_attempts = db.Column(db.Integer, default=0) + value = db.Column(db.Integer) + category = db.Column(db.String(80)) + type = db.Column(db.String(80)) + hidden = db.Column(db.Boolean) + __mapper_args__ = { + 'polymorphic_identity': 'standard', + 'polymorphic_on': type + } + + def __init__(self, name, description, value, category, type='standard'): + self.name = name + self.description = description + self.value = value + self.category = category + self.type = type + + def __repr__(self): + return '' % self.name + + +class Hints(db.Model): + id = db.Column(db.Integer, primary_key=True) + type = db.Column(db.Integer, default=0) + chal = db.Column(db.Integer, db.ForeignKey('challenges.id')) + hint = db.Column(db.Text) + cost = db.Column(db.Integer, default=0) + + def __init__(self, chal, hint, cost=0, type=0): + self.chal = chal + self.hint = hint + self.cost = cost + self.type = type + + def __repr__(self): + return '' % self.hint + + +class Awards(db.Model): + id = db.Column(db.Integer, primary_key=True) + teamid = db.Column(db.Integer, db.ForeignKey('teams.id')) + name = db.Column(db.String(80)) + description = db.Column(db.Text) + date = db.Column(db.DateTime, default=datetime.datetime.utcnow) + value = db.Column(db.Integer) + category = db.Column(db.String(80)) + icon = db.Column(db.Text) + + def __init__(self, teamid, name, value): + self.teamid = teamid + self.name = name + self.value = value + + def __repr__(self): + return '' % self.name + + +class Tags(db.Model): + id = db.Column(db.Integer, primary_key=True) + chal = db.Column(db.Integer, db.ForeignKey('challenges.id')) + tag = db.Column(db.String(80)) + + def __init__(self, chal, tag): + self.chal = chal + self.tag = tag + + def __repr__(self): + return "".format(self.tag, self.chal) + + +class Files(db.Model): + id = db.Column(db.Integer, primary_key=True) + chal = db.Column(db.Integer, db.ForeignKey('challenges.id')) + location = db.Column(db.Text) + + def __init__(self, chal, location): + self.chal = chal + self.location = location + + def __repr__(self): + return "".format(self.location, self.chal) + + +class Keys(db.Model): + id = db.Column(db.Integer, primary_key=True) + chal = db.Column(db.Integer, db.ForeignKey('challenges.id')) + type = db.Column(db.String(80)) + flag = db.Column(db.Text) + data = db.Column(db.Text) + + def __init__(self, chal, flag, type): + self.chal = chal + self.flag = flag + self.type = type + + def __repr__(self): + return "".format(self.flag, self.chal) + + +class Teams(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(128), unique=True) + email = db.Column(db.String(128), unique=True) + password = db.Column(db.String(128)) + website = db.Column(db.String(128)) + affiliation = db.Column(db.String(128)) + country = db.Column(db.String(32)) + bracket = db.Column(db.String(32)) + banned = db.Column(db.Boolean, default=False) + verified = db.Column(db.Boolean, default=False) + admin = db.Column(db.Boolean, default=False) + joined = db.Column(db.DateTime, default=datetime.datetime.utcnow) + + def __init__(self, name, email, password): + self.name = name + self.email = email + self.password = bcrypt_sha256.encrypt(str(password)) + + def __repr__(self): + return '' % self.name + + def score(self, admin=False): + score = db.func.sum(Challenges.value).label('score') + team = db.session.query(Solves.teamid, score).join(Teams).join(Challenges).filter(Teams.id == self.id) + award_score = db.func.sum(Awards.value).label('award_score') + award = db.session.query(award_score).filter_by(teamid=self.id) + + if not admin: + freeze = Config.query.filter_by(key='freeze').first() + if freeze and freeze.value: + freeze = int(freeze.value) + freeze = datetime.datetime.utcfromtimestamp(freeze) + team = team.filter(Solves.date < freeze) + award = award.filter(Awards.date < freeze) + + team = team.group_by(Solves.teamid).first() + award = award.first() + + if team and award: + return int(team.score or 0) + int(award.award_score or 0) + elif team: + return int(team.score or 0) + elif award: + return int(award.award_score or 0) + else: + return 0 + + def place(self, admin=False): + """ + This method is generally a clone of CTFd.scoreboard.get_standings. + The point being that models.py must be self-reliant and have little + to no imports within the CTFd application as importing from the + application itself will result in a circular import. + """ + scores = db.session.query( + Solves.teamid.label('teamid'), + db.func.sum(Challenges.value).label('score'), + db.func.max(Solves.id).label('id'), + db.func.max(Solves.date).label('date') + ).join(Challenges).filter(Challenges.value != 0).group_by(Solves.teamid) + + awards = db.session.query( + Awards.teamid.label('teamid'), + db.func.sum(Awards.value).label('score'), + db.func.max(Awards.id).label('id'), + db.func.max(Awards.date).label('date') + ).filter(Awards.value != 0).group_by(Awards.teamid) + + if not admin: + freeze = Config.query.filter_by(key='freeze').first() + if freeze and freeze.value: + freeze = int(freeze.value) + freeze = datetime.datetime.utcfromtimestamp(freeze) + scores = scores.filter(Solves.date < freeze) + awards = awards.filter(Awards.date < freeze) + + results = union_all(scores, awards).alias('results') + + sumscores = db.session.query( + results.columns.teamid, + db.func.sum(results.columns.score).label('score'), + db.func.max(results.columns.id).label('id'), + db.func.max(results.columns.date).label('date') + ).group_by(results.columns.teamid).subquery() + + if admin: + standings_query = db.session.query( + Teams.id.label('teamid'), + ) \ + .join(sumscores, Teams.id == sumscores.columns.teamid) \ + .order_by(sumscores.columns.score.desc(), sumscores.columns.id) + else: + standings_query = db.session.query( + Teams.id.label('teamid'), + ) \ + .join(sumscores, Teams.id == sumscores.columns.teamid) \ + .filter(Teams.banned == False) \ + .order_by(sumscores.columns.score.desc(), sumscores.columns.id) + + standings = standings_query.all() + + # http://codegolf.stackexchange.com/a/4712 + try: + i = standings.index((self.id,)) + 1 + k = i % 10 + return "%d%s" % (i, "tsnrhtdd"[(i / 10 % 10 != 1) * (k < 4) * k::4]) + except ValueError: + return 0 + + +class Solves(db.Model): + __table_args__ = (db.UniqueConstraint('chalid', 'teamid'), {}) + id = db.Column(db.Integer, primary_key=True) + chalid = db.Column(db.Integer, db.ForeignKey('challenges.id')) + teamid = db.Column(db.Integer, db.ForeignKey('teams.id')) + ip = db.Column(db.String(46)) + flag = db.Column(db.Text) + date = db.Column(db.DateTime, default=datetime.datetime.utcnow) + team = db.relationship('Teams', foreign_keys="Solves.teamid", lazy='joined') + chal = db.relationship('Challenges', foreign_keys="Solves.chalid", lazy='joined') + + # value = db.Column(db.Integer) + + def __init__(self, teamid, chalid, ip, flag): + self.ip = ip + self.chalid = chalid + self.teamid = teamid + self.flag = flag + # self.value = value + + def __repr__(self): + return ''.format(self.teamid, self.chalid, self.ip, self.flag) + + +class WrongKeys(db.Model): + id = db.Column(db.Integer, primary_key=True) + chalid = db.Column(db.Integer, db.ForeignKey('challenges.id')) + teamid = db.Column(db.Integer, db.ForeignKey('teams.id')) + ip = db.Column(db.String(46)) + date = db.Column(db.DateTime, default=datetime.datetime.utcnow) + flag = db.Column(db.Text) + chal = db.relationship('Challenges', foreign_keys="WrongKeys.chalid", lazy='joined') + + def __init__(self, teamid, chalid, ip, flag): + self.ip = ip + self.teamid = teamid + self.chalid = chalid + self.flag = flag + + def __repr__(self): + return ''.format(self.teamid, self.chalid, self.ip, self.flag) + + +class Unlocks(db.Model): + id = db.Column(db.Integer, primary_key=True) + teamid = db.Column(db.Integer, db.ForeignKey('teams.id')) + date = db.Column(db.DateTime, default=datetime.datetime.utcnow) + itemid = db.Column(db.Integer) + model = db.Column(db.String(32)) + + def __init__(self, model, teamid, itemid): + self.model = model + self.teamid = teamid + self.itemid = itemid + + def __repr__(self): + return '' % self.teamid + + +class Tracking(db.Model): + id = db.Column(db.Integer, primary_key=True) + ip = db.Column(db.String(46)) + team = db.Column(db.Integer, db.ForeignKey('teams.id')) + date = db.Column(db.DateTime, default=datetime.datetime.utcnow) + + def __init__(self, ip, team): + self.ip = ip + self.team = team + + def __repr__(self): + return '' % self.team + + +class Config(db.Model): + id = db.Column(db.Integer, primary_key=True) + key = db.Column(db.Text) + value = db.Column(db.Text) + + def __init__(self, key, value): + self.key = key + self.value = value + + +if __name__ == '__main__': + print("/*\\ Migrating your database to 2.0.0 can potentially lose data./*\\") + print("""/*\\ Please be sure to back up all data by: + * creating a CTFd export + * creating a dump of your actual database + * and backing up the CTFd source code directory""") + print("/*\\ CTFd maintainers are not responsible for any data loss! /*\\") + if input('Run database migrations (Y/N)').lower().strip() == 'y': + pass + else: + print('/*\\ Aborting database migrations... /*\\') + print('/*\\ Exiting... /*\\') + exit(1) + + db_url = config.Config.SQLALCHEMY_DATABASE_URI + old_data = {} + + old_conn = dataset.connect(config.Config.SQLALCHEMY_DATABASE_URI) + tables = old_conn.tables + for table in tables: + old_data[table] = old_conn[table].all() + + if 'alembic_version' in old_data: + old_data.pop('alembic_version') + + print('Current Tables:') + for table in old_data.keys(): + print(table) + + old_conn.executable.close() + + print('DROPPING DATABASE') + drop_database(db_url) + + app = create_app() + + new_conn = dataset.connect(config.Config.SQLALCHEMY_DATABASE_URI) + + print('MIGRATING Challenges') + for challenge in old_data['challenges']: + hidden = challenge.pop('hidden') + challenge['state'] = 'hidden' if hidden else 'visible' + new_conn['challenges'].insert(dict(challenge)) + del old_data['challenges'] + + print('MIGRATING Teams') + for team in old_data['teams']: + admin = team.pop('admin') + team['type'] = 'admin' if admin else 'user' + team['hidden'] = bool(team.pop('banned')) + team['banned'] = False + team['verified'] = bool(team.pop('verified')) + new_conn['users'].insert(dict(team)) + del old_data['teams'] + + print('MIGRATING Pages') + for page in old_data['pages']: + page['content'] = page.pop('html') + new_conn['pages'].insert(dict(page)) + del old_data['pages'] + + print('MIGRATING Keys') + for key in old_data['keys']: + key['challenge_id'] = key.pop('chal') + key['content'] = key.pop('flag') + new_conn['flags'].insert(dict(key)) + del old_data['keys'] + + print('MIGRATING Tags') + for tag in old_data['tags']: + tag['challenge_id'] = tag.pop('chal') + tag['value'] = tag.pop('tag') + new_conn['tags'].insert(dict(tag)) + del old_data['tags'] + + print('MIGRATING Files') + for f in old_data['files']: + challenge_id = f.pop('chal') + if challenge_id: + f['challenge_id'] = challenge_id + f['type'] = 'challenge' + else: + f['page_id'] = None + f['type'] = 'page' + new_conn['files'].insert(dict(f)) + del old_data['files'] + + print('MIGRATING Hints') + for hint in old_data['hints']: + hint['type'] = 'standard' + hint['challenge_id'] = hint.pop('chal') + hint['content'] = hint.pop('hint') + new_conn['hints'].insert(dict(hint)) + del old_data['hints'] + + print('MIGRATING Unlocks') + for unlock in old_data['unlocks']: + unlock['user_id'] = unlock.pop('teamid') # This is intentional as previous CTFds are effectively in user mode + unlock['target'] = unlock.pop('item_id') + unlock['type'] = unlock.pop('model') + new_conn['unlocks'].insert(dict(unlock)) + del old_data['unlocks'] + + print('MIGRATING Awards') + for award in old_data['awards']: + award['user_id'] = award.pop('teamid') # This is intentional as previous CTFds are effectively in user mode + new_conn['awards'].insert(dict(award)) + del old_data['awards'] + + submissions = [] + for solve in old_data['solves']: + solve.pop('id') # ID of a solve doesn't really matter + solve['challenge_id'] = solve.pop('chalid') + solve['user_id'] = solve.pop('teamid') + solve['provided'] = solve.pop('flag') + solve['type'] = 'correct' + solve['model'] = 'solve' + submissions.append(solve) + + for wrong_key in old_data['wrong_keys']: + wrong_key.pop('id') # ID of a fail doesn't really matter. + wrong_key['challenge_id'] = wrong_key.pop('chalid') + wrong_key['user_id'] = wrong_key.pop('teamid') + wrong_key['provided'] = wrong_key.pop('flag') + wrong_key['type'] = 'incorrect' + wrong_key['model'] = 'wrong_key' + submissions.append(wrong_key) + + submissions = sorted(submissions, key=lambda k: k['date']) + print('MIGRATING Solves & WrongKeys') + for submission in submissions: + model = submission.pop('model') + if model == 'solve': + new_id = new_conn['submissions'].insert(dict(submission)) + submission['id'] = new_id + new_conn['solves'].insert(dict(submission)) + elif model == 'wrong_key': + new_conn['submissions'].insert(dict(submission)) + del old_data['solves'] + del old_data['wrong_keys'] + + print('MIGRATING Tracking') + for tracking in old_data['tracking']: + tracking['user_id'] = tracking.pop('team') + new_conn['tracking'].insert(dict(tracking)) + del old_data['tracking'] + + print('MIGRATING Config') + banned = [ + 'ctf_version', + 'setup' + ] + for config in old_data['config']: + config.pop('id') + if config['key'] not in banned: + new_conn['config'].insert(dict(config)) + new_conn['config'].insert({ + 'key': 'user_mode', + 'value': 'users' + }) + del old_data['config'] + + print('MIGRATING extra tables') + for table in old_data.keys(): + print('MIGRATING', table) + data = old_data[table] + for row in data: + new_conn[table].insert(dict(row))