From 593fed300b8cf887bfa8682cec5bfaa523bd3898 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 27 Apr 2020 01:02:52 -0400 Subject: [PATCH 1/3] Remove datafreeze dependency --- CTFd/utils/exports/__init__.py | 52 ++----------------------------- CTFd/utils/exports/encoders.py | 11 +++++++ CTFd/utils/exports/freeze.py | 11 +++++++ CTFd/utils/exports/serializers.py | 44 ++++++++++++++++++++++++++ requirements.txt | 3 -- 5 files changed, 68 insertions(+), 53 deletions(-) create mode 100644 CTFd/utils/exports/encoders.py create mode 100644 CTFd/utils/exports/freeze.py create mode 100644 CTFd/utils/exports/serializers.py diff --git a/CTFd/utils/exports/__init__.py b/CTFd/utils/exports/__init__.py index 5092a571..0fc162d3 100644 --- a/CTFd/utils/exports/__init__.py +++ b/CTFd/utils/exports/__init__.py @@ -5,12 +5,9 @@ import re import tempfile import zipfile -import datafreeze import dataset import six from alembic.util import CommandError -from datafreeze.format import SERIALIZERS -from datafreeze.format.fjson import JSONEncoder, JSONSerializer from flask import current_app as app from flask_migrate import upgrade from sqlalchemy.exc import OperationalError, ProgrammingError @@ -20,6 +17,7 @@ from CTFd import __version__ as CTFD_VERSION from CTFd.cache import cache from CTFd.models import db, get_class_by_tablename from CTFd.utils import get_app_config, set_config +from CTFd.utils.exports.freeze import freeze_export from CTFd.utils.migrations import ( create_database, drop_database, @@ -29,52 +27,6 @@ from CTFd.utils.migrations import ( from CTFd.utils.uploads import get_uploader -class CTFdSerializer(JSONSerializer): - """ - Slightly modified datafreeze serializer so that we can properly - export the CTFd database into a zip file. - """ - - def close(self): - for path, result in self.buckets.items(): - result = self.wrap(result) - - if self.fileobj is None: - fh = open(path, "wb") - else: - fh = self.fileobj - - # Certain databases (MariaDB) store JSON as LONGTEXT. - # Before emitting a file we should standardize to valid JSON (i.e. a dict) - # See Issue #973 - for i, r in enumerate(result["results"]): - data = r.get("requirements") - if data: - try: - if isinstance(data, six.string_types): - result["results"][i]["requirements"] = json.loads(data) - except ValueError: - pass - - data = json.dumps( - result, cls=JSONEncoder, indent=self.export.get_int("indent") - ) - - callback = self.export.get("callback") - if callback: - data = "%s && %s(%s);" % (callback, callback, data) - - if six.PY3: - fh.write(bytes(data, encoding="utf-8")) - else: - fh.write(data) - if self.fileobj is None: - fh.close() - - -SERIALIZERS["ctfd"] = CTFdSerializer # Load the custom serializer - - def export_ctf(): # TODO: For some unknown reason dataset is only able to see alembic_version during tests. # Even using a real sqlite database. This makes this test impossible to pass in sqlite. @@ -89,7 +41,7 @@ def export_ctf(): for table in tables: result = db[table].all() result_file = six.BytesIO() - datafreeze.freeze(result, format="ctfd", fileobj=result_file) + freeze_export(result, fileobj=result_file) result_file.seek(0) backup_zip.writestr("db/{}.json".format(table), result_file.read()) diff --git a/CTFd/utils/exports/encoders.py b/CTFd/utils/exports/encoders.py new file mode 100644 index 00000000..5643b9db --- /dev/null +++ b/CTFd/utils/exports/encoders.py @@ -0,0 +1,11 @@ +import json +from datetime import datetime, date +from decimal import Decimal + + +class JSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime, date)): + return obj.isoformat() + if isinstance(obj, Decimal): + return str(obj) diff --git a/CTFd/utils/exports/freeze.py b/CTFd/utils/exports/freeze.py new file mode 100644 index 00000000..23059ef3 --- /dev/null +++ b/CTFd/utils/exports/freeze.py @@ -0,0 +1,11 @@ +from CTFd.utils.exports.serializers import JSONSerializer +from sqlalchemy.exc import ProgrammingError, OperationalError + + +def freeze_export(result, fileobj): + try: + query = result + serializer = JSONSerializer(query, fileobj) + serializer.serialize() + except (OperationalError, ProgrammingError) as e: + raise OperationalError("Invalid query: %s" % e) diff --git a/CTFd/utils/exports/serializers.py b/CTFd/utils/exports/serializers.py new file mode 100644 index 00000000..51580972 --- /dev/null +++ b/CTFd/utils/exports/serializers.py @@ -0,0 +1,44 @@ +import json +import six +from collections import OrderedDict + +from CTFd.utils.exports.encoders import JSONEncoder + + +class JSONSerializer(object): + def __init__(self, query, fileobj): + self.query = query + self.fileobj = fileobj + self.buckets = [] + + def serialize(self): + for row in self.query: + self.write(None, row) + self.close() + + def write(self, path, result): + self.buckets.append([result]) + + def wrap(self, result): + result = OrderedDict([("count", len(result)), ("results", result)]) + result["meta"] = {} + return result + + def close(self): + for result in self.buckets: + result = self.wrap(result) + + # Certain databases (MariaDB) store JSON as LONGTEXT. + # Before emitting a file we should standardize to valid JSON (i.e. a dict) + # See Issue #973 + for i, r in enumerate(result["results"]): + data = r.get("requirements") + if data: + try: + if isinstance(data, six.string_types): + result["results"][i]["requirements"] = json.loads(data) + except ValueError: + pass + + data = json.dumps(result, cls=JSONEncoder, indent=2) + self.fileobj.write(data.encode("utf-8")) diff --git a/requirements.txt b/requirements.txt index a4d82294..98816258 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,13 +13,10 @@ itsdangerous==1.1.0 requests>=2.20.0 PyMySQL==0.9.3 gunicorn==19.9.0 -banal==0.4.2 -normality==2.0.0 dataset==1.1.2 mistune==0.8.4 netaddr==0.7.19 redis==3.3.11 -datafreeze==0.1.0 gevent==1.4.0 python-dotenv==0.10.3 flask-restx==0.1.1 From 89ade49695c8961f23879050dda8aa684603c04b Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 27 Apr 2020 01:49:56 -0400 Subject: [PATCH 2/3] Fix serializer --- CTFd/utils/exports/serializers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CTFd/utils/exports/serializers.py b/CTFd/utils/exports/serializers.py index 51580972..de161f22 100644 --- a/CTFd/utils/exports/serializers.py +++ b/CTFd/utils/exports/serializers.py @@ -1,6 +1,6 @@ import json import six -from collections import OrderedDict +from collections import defaultdict, OrderedDict from CTFd.utils.exports.encoders import JSONEncoder @@ -9,7 +9,7 @@ class JSONSerializer(object): def __init__(self, query, fileobj): self.query = query self.fileobj = fileobj - self.buckets = [] + self.buckets = defaultdict(list) def serialize(self): for row in self.query: @@ -17,7 +17,7 @@ class JSONSerializer(object): self.close() def write(self, path, result): - self.buckets.append([result]) + self.buckets[path].append(result) def wrap(self, result): result = OrderedDict([("count", len(result)), ("results", result)]) @@ -25,7 +25,7 @@ class JSONSerializer(object): return result def close(self): - for result in self.buckets: + for path, result in self.buckets.items(): result = self.wrap(result) # Certain databases (MariaDB) store JSON as LONGTEXT. From 41a0ebc68da0e55514637e96f6e49e8e9612eb6a Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 27 Apr 2020 02:00:11 -0400 Subject: [PATCH 3/3] Make minified JSON --- CTFd/utils/exports/encoders.py | 11 ----------- CTFd/utils/exports/serializers.py | 17 +++++++++++++---- 2 files changed, 13 insertions(+), 15 deletions(-) delete mode 100644 CTFd/utils/exports/encoders.py diff --git a/CTFd/utils/exports/encoders.py b/CTFd/utils/exports/encoders.py deleted file mode 100644 index 5643b9db..00000000 --- a/CTFd/utils/exports/encoders.py +++ /dev/null @@ -1,11 +0,0 @@ -import json -from datetime import datetime, date -from decimal import Decimal - - -class JSONEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, (datetime, date)): - return obj.isoformat() - if isinstance(obj, Decimal): - return str(obj) diff --git a/CTFd/utils/exports/serializers.py b/CTFd/utils/exports/serializers.py index de161f22..6526265e 100644 --- a/CTFd/utils/exports/serializers.py +++ b/CTFd/utils/exports/serializers.py @@ -1,8 +1,16 @@ import json import six from collections import defaultdict, OrderedDict +from datetime import datetime, date +from decimal import Decimal -from CTFd.utils.exports.encoders import JSONEncoder + +class JSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime, date)): + return obj.isoformat() + if isinstance(obj, Decimal): + return str(obj) class JSONSerializer(object): @@ -20,8 +28,9 @@ class JSONSerializer(object): self.buckets[path].append(result) def wrap(self, result): - result = OrderedDict([("count", len(result)), ("results", result)]) - result["meta"] = {} + result = OrderedDict( + [("count", len(result)), ("results", result), ("meta", {})] + ) return result def close(self): @@ -40,5 +49,5 @@ class JSONSerializer(object): except ValueError: pass - data = json.dumps(result, cls=JSONEncoder, indent=2) + data = json.dumps(result, cls=JSONEncoder, separators=(",", ":")) self.fileobj.write(data.encode("utf-8"))