From 817b67d1b05235799a0ea5875a4db882b529c750 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 29 Apr 2020 18:30:17 -0400 Subject: [PATCH 1/7] Start to refactor tracker to cache user IPs --- CTFd/utils/initialization/__init__.py | 20 ++++++++++++++++---- CTFd/utils/user/__init__.py | 12 +++++++++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/CTFd/utils/initialization/__init__.py b/CTFd/utils/initialization/__init__.py index 158cb870..fa6c6cbb 100644 --- a/CTFd/utils/initialization/__init__.py +++ b/CTFd/utils/initialization/__init__.py @@ -38,7 +38,14 @@ from CTFd.utils.plugins import ( ) from CTFd.utils.security.auth import login_user, logout_user, lookup_user_token from CTFd.utils.security.csrf import generate_nonce -from CTFd.utils.user import authed, get_current_team, get_current_user, get_ip, is_admin +from CTFd.utils.user import ( + authed, + get_current_team, + get_current_user, + get_ip, + get_user_ips, + is_admin, +) def init_template_filters(app): @@ -170,12 +177,17 @@ def init_request_processors(app): return if authed(): - track = Tracking.query.filter_by(ip=get_ip(), user_id=session["id"]).first() - if not track: + user_ips = get_user_ips(user_id=session["id"]) + ip = get_ip() + if ip not in user_ips: visit = Tracking(ip=get_ip(), user_id=session["id"]) db.session.add(visit) else: - track.date = datetime.datetime.utcnow() + if request.method != "GET": + track = Tracking.query.filter_by( + ip=get_ip(), user_id=session["id"] + ).first() + track.date = datetime.datetime.utcnow() try: db.session.commit() diff --git a/CTFd/utils/user/__init__.py b/CTFd/utils/user/__init__.py index 81762ba6..8f458248 100644 --- a/CTFd/utils/user/__init__.py +++ b/CTFd/utils/user/__init__.py @@ -4,7 +4,8 @@ import re from flask import current_app as app from flask import request, session -from CTFd.models import Fails, Users, db +from CTFd.cache import cache +from CTFd.models import Fails, Users, db, Tracking from CTFd.utils import get_config @@ -80,6 +81,15 @@ def get_ip(req=None): return remote_addr +def get_user_ips(user_id): + addrs = ( + Tracking.query.with_entities(Tracking.ip.distinct()) + .filter_by(user_id=user_id) + .all() + ) + return [ip for ip, in addrs] + + def get_wrong_submissions_per_minute(account_id): """ Get incorrect submissions per minute. From 541d7e9cfa153fc1a788eddef2e40958b36ce91a Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 29 Apr 2020 22:34:33 -0400 Subject: [PATCH 2/7] Add a wrapper for get_user_ips --- CTFd/utils/initialization/__init__.py | 4 ++-- CTFd/utils/user/__init__.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CTFd/utils/initialization/__init__.py b/CTFd/utils/initialization/__init__.py index fa6c6cbb..0cafbef7 100644 --- a/CTFd/utils/initialization/__init__.py +++ b/CTFd/utils/initialization/__init__.py @@ -43,7 +43,7 @@ from CTFd.utils.user import ( get_current_team, get_current_user, get_ip, - get_user_ips, + get_current_user_ips, is_admin, ) @@ -177,7 +177,7 @@ def init_request_processors(app): return if authed(): - user_ips = get_user_ips(user_id=session["id"]) + user_ips = get_current_user_ips() ip = get_ip() if ip not in user_ips: visit = Tracking(ip=get_ip(), user_id=session["id"]) diff --git a/CTFd/utils/user/__init__.py b/CTFd/utils/user/__init__.py index 8f458248..188e98db 100644 --- a/CTFd/utils/user/__init__.py +++ b/CTFd/utils/user/__init__.py @@ -81,6 +81,14 @@ def get_ip(req=None): return remote_addr +def get_current_user_ips(): + if authed(): + return get_user_ips(user_id=session["id"]) + else: + return None + + +@cache.memoize def get_user_ips(user_id): addrs = ( Tracking.query.with_entities(Tracking.ip.distinct()) From 4e28e30cb8492f1cc5d1e608a642255c8ad7436b Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 29 Apr 2020 23:10:39 -0400 Subject: [PATCH 3/7] Fix memoize call --- CTFd/utils/user/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CTFd/utils/user/__init__.py b/CTFd/utils/user/__init__.py index 188e98db..59c1a40e 100644 --- a/CTFd/utils/user/__init__.py +++ b/CTFd/utils/user/__init__.py @@ -88,7 +88,7 @@ def get_current_user_ips(): return None -@cache.memoize +@cache.memoize(timeout=60) def get_user_ips(user_id): addrs = ( Tracking.query.with_entities(Tracking.ip.distinct()) From 87895918f5e8464c9d6ebedbe24fb97d21a61aca Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 30 Apr 2020 02:45:12 -0400 Subject: [PATCH 4/7] Fix tracker code to fix tests --- CTFd/cache/__init__.py | 6 ++++++ CTFd/utils/initialization/__init__.py | 18 +++++++++++------- tests/admin/test_config.py | 3 +++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/CTFd/cache/__init__.py b/CTFd/cache/__init__.py index 78a63b59..601616a0 100644 --- a/CTFd/cache/__init__.py +++ b/CTFd/cache/__init__.py @@ -44,3 +44,9 @@ def clear_pages(): cache.delete_memoized(get_pages) cache.delete_memoized(get_page) + + +def clear_user_ips(user_id): + from CTFd.utils.user import get_user_ips + + cache.delete_memoized(get_user_ips, user_id=user_id) diff --git a/CTFd/utils/initialization/__init__.py b/CTFd/utils/initialization/__init__.py index 0cafbef7..7e797215 100644 --- a/CTFd/utils/initialization/__init__.py +++ b/CTFd/utils/initialization/__init__.py @@ -7,6 +7,7 @@ from flask import abort, redirect, render_template, request, session, url_for from sqlalchemy.exc import IntegrityError, InvalidRequestError from werkzeug.wsgi import DispatcherMiddleware +from CTFd.cache import clear_user_ips from CTFd.exceptions import UserNotFoundException, UserTokenExpiredException from CTFd.models import Tracking, db from CTFd.utils import config, get_config, markdown @@ -179,9 +180,10 @@ def init_request_processors(app): if authed(): user_ips = get_current_user_ips() ip = get_ip() + track = None if ip not in user_ips: - visit = Tracking(ip=get_ip(), user_id=session["id"]) - db.session.add(visit) + track = Tracking(ip=get_ip(), user_id=session["id"]) + db.session.add(track) else: if request.method != "GET": track = Tracking.query.filter_by( @@ -189,11 +191,13 @@ def init_request_processors(app): ).first() track.date = datetime.datetime.utcnow() - try: - db.session.commit() - except (InvalidRequestError, IntegrityError): - db.session.rollback() - logout_user() + if track: + try: + db.session.commit() + except (InvalidRequestError, IntegrityError): + db.session.rollback() + logout_user() + clear_user_ips(user_id=session["id"]) if authed(): user = get_current_user() diff --git a/tests/admin/test_config.py b/tests/admin/test_config.py index d33d7910..dd0da3a5 100644 --- a/tests/admin/test_config.py +++ b/tests/admin/test_config.py @@ -58,6 +58,9 @@ def test_reset(): gen_fail(app.db, user_id=user_obj.id, challenge_id=random.randint(1, 10)) gen_tracking(app.db, user_id=user_obj.id) + print("eet") + print(Tracking.query.all()) + # Add PageFiles for x in range(5): gen_file( From b3b654bf03a86d0c15ebbbc91445b41577064a1e Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 30 Apr 2020 03:21:13 -0400 Subject: [PATCH 5/7] Rebase 2.4.0-dev --- CTFd/utils/initialization/__init__.py | 2 +- CTFd/utils/user/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CTFd/utils/initialization/__init__.py b/CTFd/utils/initialization/__init__.py index bcbfcadd..4b496865 100644 --- a/CTFd/utils/initialization/__init__.py +++ b/CTFd/utils/initialization/__init__.py @@ -42,6 +42,7 @@ from CTFd.utils.security.csrf import generate_nonce from CTFd.utils.user import ( authed, get_current_user_attrs, + get_current_user_ips, get_current_team_attrs, get_ip, is_admin, @@ -201,7 +202,6 @@ def init_request_processors(app): logout_user() clear_user_ips(user_id=session["id"]) - db.session.close() @app.before_request def banned(): diff --git a/CTFd/utils/user/__init__.py b/CTFd/utils/user/__init__.py index 9a026a5d..5186e2b0 100644 --- a/CTFd/utils/user/__init__.py +++ b/CTFd/utils/user/__init__.py @@ -7,7 +7,7 @@ from flask import request, session from CTFd.cache import cache from CTFd.constants.users import UserAttrs from CTFd.constants.teams import TeamAttrs -from CTFd.models import Fails, Users, db, Teams +from CTFd.models import Fails, Users, db, Teams, Tracking from CTFd.utils import get_config From c69633b4791bc1b8b87870e8259a0e7d87085873 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 30 Apr 2020 03:33:55 -0400 Subject: [PATCH 6/7] Remove testing data --- tests/admin/test_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/admin/test_config.py b/tests/admin/test_config.py index dd0da3a5..d33d7910 100644 --- a/tests/admin/test_config.py +++ b/tests/admin/test_config.py @@ -58,9 +58,6 @@ def test_reset(): gen_fail(app.db, user_id=user_obj.id, challenge_id=random.randint(1, 10)) gen_tracking(app.db, user_id=user_obj.id) - print("eet") - print(Tracking.query.all()) - # Add PageFiles for x in range(5): gen_file( From 2f1b3265f8142c26e8eb6b6b57be40531d0636b9 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 30 Apr 2020 14:39:42 -0400 Subject: [PATCH 7/7] Run make lint --- CTFd/utils/initialization/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/CTFd/utils/initialization/__init__.py b/CTFd/utils/initialization/__init__.py index 4b496865..19aff5d9 100644 --- a/CTFd/utils/initialization/__init__.py +++ b/CTFd/utils/initialization/__init__.py @@ -202,7 +202,6 @@ def init_request_processors(app): logout_user() clear_user_ips(user_id=session["id"]) - @app.before_request def banned(): if request.endpoint == "views.themes":