From f24f2a18bb262bb57babc1ee7cf27bf65e3ab3d0 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Fri, 8 Apr 2022 16:52:04 -0400 Subject: [PATCH] Import backup improvements (#2078) * Add progress tracking to import_ctf * Make imports happen in the background so that we can see status * Add GET /admin/import to see status of import * Disable the public interface during imports * Closes #1980 --- CTFd/admin/__init__.py | 33 +++++----- CTFd/themes/admin/assets/js/pages/configs.js | 7 +-- .../admin/static/js/pages/configs.dev.js | 2 +- .../admin/static/js/pages/configs.min.js | 2 +- CTFd/themes/admin/templates/import.html | 50 +++++++++++++++ CTFd/utils/__init__.py | 11 ++++ CTFd/utils/exports/__init__.py | 63 ++++++++++++++++++- CTFd/utils/initialization/__init__.py | 8 ++- 8 files changed, 151 insertions(+), 25 deletions(-) create mode 100644 CTFd/themes/admin/templates/import.html diff --git a/CTFd/admin/__init__.py b/CTFd/admin/__init__.py index 51aa5078..20f96ef6 100644 --- a/CTFd/admin/__init__.py +++ b/CTFd/admin/__init__.py @@ -45,9 +45,8 @@ from CTFd.utils import config as ctf_config from CTFd.utils import get_config, set_config from CTFd.utils.csv import dump_csv, load_challenges_csv, load_teams_csv, load_users_csv from CTFd.utils.decorators import admins_only +from CTFd.utils.exports import background_import_ctf from CTFd.utils.exports import export_ctf as export_ctf_util -from CTFd.utils.exports import import_ctf as import_ctf_util -from CTFd.utils.helpers import get_errors from CTFd.utils.security.auth import logout_user from CTFd.utils.uploads import delete_file from CTFd.utils.user import is_admin @@ -88,21 +87,25 @@ def plugin(plugin): return "1" -@admin.route("/admin/import", methods=["POST"]) +@admin.route("/admin/import", methods=["GET", "POST"]) @admins_only def import_ctf(): - backup = request.files["backup"] - errors = get_errors() - try: - import_ctf_util(backup) - except Exception as e: - print(e) - errors.append(repr(e)) - - if errors: - return errors[0], 500 - else: - return redirect(url_for("admin.config")) + if request.method == "GET": + start_time = cache.get("import_start_time") + end_time = cache.get("import_end_time") + import_status = cache.get("import_status") + import_error = cache.get("import_error") + return render_template( + "admin/import.html", + start_time=start_time, + end_time=end_time, + import_status=import_status, + import_error=import_error, + ) + elif request.method == "POST": + backup = request.files["backup"] + background_import_ctf(backup) + return redirect(url_for("admin.import_ctf")) @admin.route("/admin/export", methods=["GET", "POST"]) diff --git a/CTFd/themes/admin/assets/js/pages/configs.js b/CTFd/themes/admin/assets/js/pages/configs.js index b0d2e489..5bc4be76 100644 --- a/CTFd/themes/admin/assets/js/pages/configs.js +++ b/CTFd/themes/admin/assets/js/pages/configs.js @@ -359,12 +359,7 @@ function importConfig(event) { target: pg, width: 100 }); - setTimeout(function() { - pg.modal("hide"); - }, 500); - setTimeout(function() { - window.location.reload(); - }, 700); + location.href = CTFd.config.urlRoot + "/admin/import"; } }); } diff --git a/CTFd/themes/admin/static/js/pages/configs.dev.js b/CTFd/themes/admin/static/js/pages/configs.dev.js index f91b0600..8492b61b 100644 --- a/CTFd/themes/admin/static/js/pages/configs.dev.js +++ b/CTFd/themes/admin/static/js/pages/configs.dev.js @@ -162,7 +162,7 @@ /***/ (function(module, exports, __webpack_require__) { ; -eval("\n\n__webpack_require__(/*! ./main */ \"./CTFd/themes/admin/assets/js/pages/main.js\");\n\n__webpack_require__(/*! core/utils */ \"./CTFd/themes/core/assets/js/utils.js\");\n\n__webpack_require__(/*! bootstrap/js/dist/tab */ \"./node_modules/bootstrap/js/dist/tab.js\");\n\nvar _dayjs = _interopRequireDefault(__webpack_require__(/*! dayjs */ \"./node_modules/dayjs/dayjs.min.js\"));\n\nvar _advancedFormat = _interopRequireDefault(__webpack_require__(/*! dayjs/plugin/advancedFormat */ \"./node_modules/dayjs/plugin/advancedFormat.js\"));\n\nvar _utc = _interopRequireDefault(__webpack_require__(/*! dayjs/plugin/utc */ \"./node_modules/dayjs/plugin/utc.js\"));\n\nvar _timezone = _interopRequireDefault(__webpack_require__(/*! dayjs/plugin/timezone */ \"./node_modules/dayjs/plugin/timezone.js\"));\n\nvar _timezones = _interopRequireDefault(__webpack_require__(/*! ../timezones */ \"./CTFd/themes/admin/assets/js/timezones.js\"));\n\nvar _CTFd = _interopRequireDefault(__webpack_require__(/*! core/CTFd */ \"./CTFd/themes/core/assets/js/CTFd.js\"));\n\nvar _helpers = _interopRequireDefault(__webpack_require__(/*! core/helpers */ \"./CTFd/themes/core/assets/js/helpers.js\"));\n\nvar _jquery = _interopRequireDefault(__webpack_require__(/*! jquery */ \"./node_modules/jquery/dist/jquery.js\"));\n\nvar _ezq = __webpack_require__(/*! core/ezq */ \"./CTFd/themes/core/assets/js/ezq.js\");\n\nvar _codemirror = _interopRequireDefault(__webpack_require__(/*! codemirror */ \"./node_modules/codemirror/lib/codemirror.js\"));\n\n__webpack_require__(/*! codemirror/mode/htmlmixed/htmlmixed.js */ \"./node_modules/codemirror/mode/htmlmixed/htmlmixed.js\");\n\nvar _vueEsm = _interopRequireDefault(__webpack_require__(/*! vue/dist/vue.esm.browser */ \"./node_modules/vue/dist/vue.esm.browser.js\"));\n\nvar _FieldList = _interopRequireDefault(__webpack_require__(/*! ../components/configs/fields/FieldList.vue */ \"./CTFd/themes/admin/assets/js/components/configs/fields/FieldList.vue\"));\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { \"default\": obj }; }\n\n_dayjs[\"default\"].extend(_advancedFormat[\"default\"]);\n\n_dayjs[\"default\"].extend(_utc[\"default\"]);\n\n_dayjs[\"default\"].extend(_timezone[\"default\"]);\n\nfunction loadTimestamp(place, timestamp) {\n if (typeof timestamp == \"string\") {\n timestamp = parseInt(timestamp, 10) * 1000;\n }\n\n var d = (0, _dayjs[\"default\"])(timestamp);\n (0, _jquery[\"default\"])(\"#\" + place + \"-month\").val(d.month() + 1); // Months are zero indexed (https://day.js.org/docs/en/get-set/month)\n\n (0, _jquery[\"default\"])(\"#\" + place + \"-day\").val(d.date());\n (0, _jquery[\"default\"])(\"#\" + place + \"-year\").val(d.year());\n (0, _jquery[\"default\"])(\"#\" + place + \"-hour\").val(d.hour());\n (0, _jquery[\"default\"])(\"#\" + place + \"-minute\").val(d.minute());\n loadDateValues(place);\n}\n\nfunction loadDateValues(place) {\n var month = (0, _jquery[\"default\"])(\"#\" + place + \"-month\").val();\n var day = (0, _jquery[\"default\"])(\"#\" + place + \"-day\").val();\n var year = (0, _jquery[\"default\"])(\"#\" + place + \"-year\").val();\n var hour = (0, _jquery[\"default\"])(\"#\" + place + \"-hour\").val();\n var minute = (0, _jquery[\"default\"])(\"#\" + place + \"-minute\").val();\n var timezone_string = (0, _jquery[\"default\"])(\"#\" + place + \"-timezone\").val();\n var utc = convertDateToMoment(month, day, year, hour, minute);\n\n if (utc.unix() && month && day && year && hour && minute) {\n (0, _jquery[\"default\"])(\"#\" + place).val(utc.unix());\n (0, _jquery[\"default\"])(\"#\" + place + \"-local\").val(utc.format(\"dddd, MMMM Do YYYY, h:mm:ss a z (zzz)\"));\n (0, _jquery[\"default\"])(\"#\" + place + \"-zonetime\").val(utc.tz(timezone_string).format(\"dddd, MMMM Do YYYY, h:mm:ss a z (zzz)\"));\n } else {\n (0, _jquery[\"default\"])(\"#\" + place).val(\"\");\n (0, _jquery[\"default\"])(\"#\" + place + \"-local\").val(\"\");\n (0, _jquery[\"default\"])(\"#\" + place + \"-zonetime\").val(\"\");\n }\n}\n\nfunction convertDateToMoment(month, day, year, hour, minute) {\n var month_num = month.toString();\n\n if (month_num.length == 1) {\n month_num = \"0\" + month_num;\n }\n\n var day_str = day.toString();\n\n if (day_str.length == 1) {\n day_str = \"0\" + day_str;\n }\n\n var hour_str = hour.toString();\n\n if (hour_str.length == 1) {\n hour_str = \"0\" + hour_str;\n }\n\n var min_str = minute.toString();\n\n if (min_str.length == 1) {\n min_str = \"0\" + min_str;\n } // 2013-02-08 24:00\n\n\n var date_string = year.toString() + \"-\" + month_num + \"-\" + day_str + \" \" + hour_str + \":\" + min_str + \":00\";\n return (0, _dayjs[\"default\"])(date_string);\n}\n\nfunction updateConfigs(event) {\n event.preventDefault();\n var obj = (0, _jquery[\"default\"])(this).serializeJSON();\n var params = {};\n\n if (obj.mail_useauth === false) {\n obj.mail_username = null;\n obj.mail_password = null;\n } else {\n if (obj.mail_username === \"\") {\n delete obj.mail_username;\n }\n\n if (obj.mail_password === \"\") {\n delete obj.mail_password;\n }\n }\n\n Object.keys(obj).forEach(function (x) {\n if (obj[x] === \"true\") {\n params[x] = true;\n } else if (obj[x] === \"false\") {\n params[x] = false;\n } else {\n params[x] = obj[x];\n }\n });\n\n _CTFd[\"default\"].api.patch_config_list({}, params).then(function (_response) {\n if (_response.success) {\n window.location.reload();\n } else {\n var errors = _response.errors.value.join(\"\\n\");\n\n (0, _ezq.ezAlert)({\n title: \"Error!\",\n body: errors,\n button: \"Okay\"\n });\n }\n });\n}\n\nfunction uploadLogo(event) {\n event.preventDefault();\n var form = event.target;\n\n _helpers[\"default\"].files.upload(form, {}, function (response) {\n var f = response.data[0];\n var params = {\n value: f.location\n };\n\n _CTFd[\"default\"].fetch(\"/api/v1/configs/ctf_logo\", {\n method: \"PATCH\",\n body: JSON.stringify(params)\n }).then(function (response) {\n return response.json();\n }).then(function (response) {\n if (response.success) {\n window.location.reload();\n } else {\n (0, _ezq.ezAlert)({\n title: \"Error!\",\n body: \"Logo uploading failed!\",\n button: \"Okay\"\n });\n }\n });\n });\n}\n\nfunction switchUserMode(event) {\n event.preventDefault();\n\n if (confirm(\"Are you sure you'd like to switch user modes?\\n\\nAll user submissions, awards, unlocks, and tracking will be deleted!\")) {\n var formData = new FormData();\n formData.append(\"submissions\", true);\n formData.append(\"nonce\", _CTFd[\"default\"].config.csrfNonce);\n fetch(_CTFd[\"default\"].config.urlRoot + \"/admin/reset\", {\n method: \"POST\",\n credentials: \"same-origin\",\n body: formData\n }); // Bind `this` so that we can reuse the updateConfigs function\n\n var binded = updateConfigs.bind(this);\n binded(event);\n }\n}\n\nfunction removeLogo() {\n (0, _ezq.ezQuery)({\n title: \"Remove logo\",\n body: \"Are you sure you'd like to remove the CTF logo?\",\n success: function success() {\n var params = {\n value: null\n };\n\n _CTFd[\"default\"].api.patch_config({\n configKey: \"ctf_logo\"\n }, params).then(function (_response) {\n window.location.reload();\n });\n }\n });\n}\n\nfunction smallIconUpload(event) {\n event.preventDefault();\n var form = event.target;\n\n _helpers[\"default\"].files.upload(form, {}, function (response) {\n var f = response.data[0];\n var params = {\n value: f.location\n };\n\n _CTFd[\"default\"].fetch(\"/api/v1/configs/ctf_small_icon\", {\n method: \"PATCH\",\n body: JSON.stringify(params)\n }).then(function (response) {\n return response.json();\n }).then(function (response) {\n if (response.success) {\n window.location.reload();\n } else {\n (0, _ezq.ezAlert)({\n title: \"Error!\",\n body: \"Icon uploading failed!\",\n button: \"Okay\"\n });\n }\n });\n });\n}\n\nfunction removeSmallIcon() {\n (0, _ezq.ezQuery)({\n title: \"Remove logo\",\n body: \"Are you sure you'd like to remove the small site icon?\",\n success: function success() {\n var params = {\n value: null\n };\n\n _CTFd[\"default\"].api.patch_config({\n configKey: \"ctf_small_icon\"\n }, params).then(function (_response) {\n window.location.reload();\n });\n }\n });\n}\n\nfunction importCSV(event) {\n event.preventDefault();\n var csv_file = document.getElementById(\"import-csv-file\").files[0];\n var csv_type = document.getElementById(\"import-csv-type\").value;\n var form_data = new FormData();\n form_data.append(\"csv_file\", csv_file);\n form_data.append(\"csv_type\", csv_type);\n form_data.append(\"nonce\", _CTFd[\"default\"].config.csrfNonce);\n var pg = (0, _ezq.ezProgressBar)({\n width: 0,\n title: \"Upload Progress\"\n });\n\n _jquery[\"default\"].ajax({\n url: _CTFd[\"default\"].config.urlRoot + \"/admin/import/csv\",\n type: \"POST\",\n data: form_data,\n processData: false,\n contentType: false,\n statusCode: {\n 500: function _(resp) {\n // Normalize errors\n var errors = JSON.parse(resp.responseText);\n var errorText = \"\";\n errors.forEach(function (element) {\n errorText += \"Line \".concat(element[0], \": \").concat(JSON.stringify(element[1]), \"\\n\");\n }); // Show errors\n\n alert(errorText); // Hide progress modal if its there\n\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: 100\n });\n setTimeout(function () {\n pg.modal(\"hide\");\n }, 500);\n }\n },\n xhr: function xhr() {\n var xhr = _jquery[\"default\"].ajaxSettings.xhr();\n\n xhr.upload.onprogress = function (e) {\n if (e.lengthComputable) {\n var width = e.loaded / e.total * 100;\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: width\n });\n }\n };\n\n return xhr;\n },\n success: function success(_data) {\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: 100\n });\n setTimeout(function () {\n pg.modal(\"hide\");\n }, 500);\n setTimeout(function () {\n window.location.reload();\n }, 700);\n }\n });\n}\n\nfunction importConfig(event) {\n event.preventDefault();\n var import_file = document.getElementById(\"import-file\").files[0];\n var form_data = new FormData();\n form_data.append(\"backup\", import_file);\n form_data.append(\"nonce\", _CTFd[\"default\"].config.csrfNonce);\n var pg = (0, _ezq.ezProgressBar)({\n width: 0,\n title: \"Upload Progress\"\n });\n\n _jquery[\"default\"].ajax({\n url: _CTFd[\"default\"].config.urlRoot + \"/admin/import\",\n type: \"POST\",\n data: form_data,\n processData: false,\n contentType: false,\n statusCode: {\n 500: function _(resp) {\n alert(resp.responseText);\n }\n },\n xhr: function xhr() {\n var xhr = _jquery[\"default\"].ajaxSettings.xhr();\n\n xhr.upload.onprogress = function (e) {\n if (e.lengthComputable) {\n var width = e.loaded / e.total * 100;\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: width\n });\n }\n };\n\n return xhr;\n },\n success: function success(_data) {\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: 100\n });\n setTimeout(function () {\n pg.modal(\"hide\");\n }, 500);\n setTimeout(function () {\n window.location.reload();\n }, 700);\n }\n });\n}\n\nfunction exportConfig(event) {\n event.preventDefault();\n window.location.href = (0, _jquery[\"default\"])(this).attr(\"href\");\n}\n\nfunction insertTimezones(target) {\n var current = (0, _jquery[\"default\"])(\"