From 2d2674aceee07c468742d9e2b48479df95089c45 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 17 Aug 2021 15:18:51 -0400 Subject: [PATCH] Improve validation and error reporting during CSV import (#1979) * Improve validation during CSV import process by using existing Marshmallow Schemas * Return CSV import errors from import functions to render out to the user * Ignore invalid fields that we can't use in Challenge CSV import --- CTFd/admin/__init__.py | 8 ++- CTFd/themes/admin/assets/js/pages/configs.js | 72 +++++++++++++++++++ .../admin/static/js/pages/configs.dev.js | 2 +- .../admin/static/js/pages/configs.min.js | 2 +- .../admin/templates/configs/backup.html | 19 +++-- CTFd/utils/csv/__init__.py | 49 ++++++++++--- 6 files changed, 134 insertions(+), 18 deletions(-) diff --git a/CTFd/admin/__init__.py b/CTFd/admin/__init__.py index dbd590a1..f31dfba7 100644 --- a/CTFd/admin/__init__.py +++ b/CTFd/admin/__init__.py @@ -6,6 +6,7 @@ from io import StringIO from flask import Blueprint, abort from flask import current_app as app from flask import ( + jsonify, redirect, render_template, render_template_string, @@ -139,8 +140,11 @@ def import_csv(): loader = loaders[csv_type] reader = csv.DictReader(csvfile) - loader(reader) - return redirect(url_for("admin.config")) + success = loader(reader) + if success is True: + return redirect(url_for("admin.config")) + else: + return jsonify(success), 500 @admin.route("/admin/export/csv") diff --git a/CTFd/themes/admin/assets/js/pages/configs.js b/CTFd/themes/admin/assets/js/pages/configs.js index ab815cd9..5b261704 100644 --- a/CTFd/themes/admin/assets/js/pages/configs.js +++ b/CTFd/themes/admin/assets/js/pages/configs.js @@ -246,6 +246,77 @@ function removeSmallIcon() { }); } +function importCSV(event) { + event.preventDefault(); + let csv_file = document.getElementById("import-csv-file").files[0]; + let csv_type = document.getElementById("import-csv-type").value; + + let form_data = new FormData(); + form_data.append("csv_file", csv_file); + form_data.append("csv_type", csv_type); + form_data.append("nonce", CTFd.config.csrfNonce); + + let pg = ezProgressBar({ + width: 0, + title: "Upload Progress" + }); + + $.ajax({ + url: CTFd.config.urlRoot + "/admin/import/csv", + type: "POST", + data: form_data, + processData: false, + contentType: false, + statusCode: { + 500: function(resp) { + // Normalize errors + let errors = JSON.parse(resp.responseText); + let errorText = ""; + errors.forEach(element => { + errorText += `Line ${element[0]}: ${JSON.stringify(element[1])}\n`; + }); + + // Show errors + alert(errorText); + + // Hide progress modal if its there + pg = ezProgressBar({ + target: pg, + width: 100 + }); + setTimeout(function() { + pg.modal("hide"); + }, 500); + } + }, + xhr: function() { + let xhr = $.ajaxSettings.xhr(); + xhr.upload.onprogress = function(e) { + if (e.lengthComputable) { + let width = (e.loaded / e.total) * 100; + pg = ezProgressBar({ + target: pg, + width: width + }); + } + }; + return xhr; + }, + success: function(_data) { + pg = ezProgressBar({ + target: pg, + width: 100 + }); + setTimeout(function() { + pg.modal("hide"); + }, 500); + setTimeout(function() { + window.location.reload(); + }, 700); + } + }); +} + function importConfig(event) { event.preventDefault(); let import_file = document.getElementById("import-file").files[0]; @@ -413,6 +484,7 @@ $(() => { $("#remove-small-icon").click(removeSmallIcon); $("#export-button").click(exportConfig); $("#import-button").click(importConfig); + $("#import-csv-form").submit(importCSV); $("#config-color-update").click(function() { const hex_code = $("#config-color-picker").val(); const user_css = theme_header_editor.getValue(); diff --git a/CTFd/themes/admin/static/js/pages/configs.dev.js b/CTFd/themes/admin/static/js/pages/configs.dev.js index efb7b40d..cd441dd6 100644 --- a/CTFd/themes/admin/static/js/pages/configs.dev.js +++ b/CTFd/themes/admin/static/js/pages/configs.dev.js @@ -162,7 +162,7 @@ /***/ (function(module, exports, __webpack_require__) { ; -eval("\n\n__webpack_require__(/*! ./main */ \"./CTFd/themes/admin/assets/js/pages/main.js\");\n\n__webpack_require__(/*! core/utils */ \"./CTFd/themes/core/assets/js/utils.js\");\n\n__webpack_require__(/*! bootstrap/js/dist/tab */ \"./node_modules/bootstrap/js/dist/tab.js\");\n\nvar _dayjs = _interopRequireDefault(__webpack_require__(/*! dayjs */ \"./node_modules/dayjs/dayjs.min.js\"));\n\nvar _advancedFormat = _interopRequireDefault(__webpack_require__(/*! dayjs/plugin/advancedFormat */ \"./node_modules/dayjs/plugin/advancedFormat.js\"));\n\nvar _utc = _interopRequireDefault(__webpack_require__(/*! dayjs/plugin/utc */ \"./node_modules/dayjs/plugin/utc.js\"));\n\nvar _timezone = _interopRequireDefault(__webpack_require__(/*! dayjs/plugin/timezone */ \"./node_modules/dayjs/plugin/timezone.js\"));\n\nvar _timezones = _interopRequireDefault(__webpack_require__(/*! ../timezones */ \"./CTFd/themes/admin/assets/js/timezones.js\"));\n\nvar _CTFd = _interopRequireDefault(__webpack_require__(/*! core/CTFd */ \"./CTFd/themes/core/assets/js/CTFd.js\"));\n\nvar _helpers = _interopRequireDefault(__webpack_require__(/*! core/helpers */ \"./CTFd/themes/core/assets/js/helpers.js\"));\n\nvar _jquery = _interopRequireDefault(__webpack_require__(/*! jquery */ \"./node_modules/jquery/dist/jquery.js\"));\n\nvar _ezq = __webpack_require__(/*! core/ezq */ \"./CTFd/themes/core/assets/js/ezq.js\");\n\nvar _codemirror = _interopRequireDefault(__webpack_require__(/*! codemirror */ \"./node_modules/codemirror/lib/codemirror.js\"));\n\n__webpack_require__(/*! codemirror/mode/htmlmixed/htmlmixed.js */ \"./node_modules/codemirror/mode/htmlmixed/htmlmixed.js\");\n\nvar _vueEsm = _interopRequireDefault(__webpack_require__(/*! vue/dist/vue.esm.browser */ \"./node_modules/vue/dist/vue.esm.browser.js\"));\n\nvar _FieldList = _interopRequireDefault(__webpack_require__(/*! ../components/configs/fields/FieldList.vue */ \"./CTFd/themes/admin/assets/js/components/configs/fields/FieldList.vue\"));\n\nfunction _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { \"default\": obj }; }\n\n_dayjs[\"default\"].extend(_advancedFormat[\"default\"]);\n\n_dayjs[\"default\"].extend(_utc[\"default\"]);\n\n_dayjs[\"default\"].extend(_timezone[\"default\"]);\n\nfunction loadTimestamp(place, timestamp) {\n if (typeof timestamp == \"string\") {\n timestamp = parseInt(timestamp, 10) * 1000;\n }\n\n var d = (0, _dayjs[\"default\"])(timestamp);\n (0, _jquery[\"default\"])(\"#\" + place + \"-month\").val(d.month() + 1); // Months are zero indexed (https://day.js.org/docs/en/get-set/month)\n\n (0, _jquery[\"default\"])(\"#\" + place + \"-day\").val(d.date());\n (0, _jquery[\"default\"])(\"#\" + place + \"-year\").val(d.year());\n (0, _jquery[\"default\"])(\"#\" + place + \"-hour\").val(d.hour());\n (0, _jquery[\"default\"])(\"#\" + place + \"-minute\").val(d.minute());\n loadDateValues(place);\n}\n\nfunction loadDateValues(place) {\n var month = (0, _jquery[\"default\"])(\"#\" + place + \"-month\").val();\n var day = (0, _jquery[\"default\"])(\"#\" + place + \"-day\").val();\n var year = (0, _jquery[\"default\"])(\"#\" + place + \"-year\").val();\n var hour = (0, _jquery[\"default\"])(\"#\" + place + \"-hour\").val();\n var minute = (0, _jquery[\"default\"])(\"#\" + place + \"-minute\").val();\n var timezone_string = (0, _jquery[\"default\"])(\"#\" + place + \"-timezone\").val();\n var utc = convertDateToMoment(month, day, year, hour, minute);\n\n if (utc.unix() && month && day && year && hour && minute) {\n (0, _jquery[\"default\"])(\"#\" + place).val(utc.unix());\n (0, _jquery[\"default\"])(\"#\" + place + \"-local\").val(utc.format(\"dddd, MMMM Do YYYY, h:mm:ss a z (zzz)\"));\n (0, _jquery[\"default\"])(\"#\" + place + \"-zonetime\").val(utc.tz(timezone_string).format(\"dddd, MMMM Do YYYY, h:mm:ss a z (zzz)\"));\n } else {\n (0, _jquery[\"default\"])(\"#\" + place).val(\"\");\n (0, _jquery[\"default\"])(\"#\" + place + \"-local\").val(\"\");\n (0, _jquery[\"default\"])(\"#\" + place + \"-zonetime\").val(\"\");\n }\n}\n\nfunction convertDateToMoment(month, day, year, hour, minute) {\n var month_num = month.toString();\n\n if (month_num.length == 1) {\n month_num = \"0\" + month_num;\n }\n\n var day_str = day.toString();\n\n if (day_str.length == 1) {\n day_str = \"0\" + day_str;\n }\n\n var hour_str = hour.toString();\n\n if (hour_str.length == 1) {\n hour_str = \"0\" + hour_str;\n }\n\n var min_str = minute.toString();\n\n if (min_str.length == 1) {\n min_str = \"0\" + min_str;\n } // 2013-02-08 24:00\n\n\n var date_string = year.toString() + \"-\" + month_num + \"-\" + day_str + \" \" + hour_str + \":\" + min_str + \":00\";\n return (0, _dayjs[\"default\"])(date_string);\n}\n\nfunction updateConfigs(event) {\n event.preventDefault();\n var obj = (0, _jquery[\"default\"])(this).serializeJSON();\n var params = {};\n\n if (obj.mail_useauth === false) {\n obj.mail_username = null;\n obj.mail_password = null;\n } else {\n if (obj.mail_username === \"\") {\n delete obj.mail_username;\n }\n\n if (obj.mail_password === \"\") {\n delete obj.mail_password;\n }\n }\n\n Object.keys(obj).forEach(function (x) {\n if (obj[x] === \"true\") {\n params[x] = true;\n } else if (obj[x] === \"false\") {\n params[x] = false;\n } else {\n params[x] = obj[x];\n }\n });\n\n _CTFd[\"default\"].api.patch_config_list({}, params).then(function (_response) {\n if (_response.success) {\n window.location.reload();\n } else {\n var errors = _response.errors.value.join(\"\\n\");\n\n (0, _ezq.ezAlert)({\n title: \"Error!\",\n body: errors,\n button: \"Okay\"\n });\n }\n });\n}\n\nfunction uploadLogo(event) {\n event.preventDefault();\n var form = event.target;\n\n _helpers[\"default\"].files.upload(form, {}, function (response) {\n var f = response.data[0];\n var params = {\n value: f.location\n };\n\n _CTFd[\"default\"].fetch(\"/api/v1/configs/ctf_logo\", {\n method: \"PATCH\",\n body: JSON.stringify(params)\n }).then(function (response) {\n return response.json();\n }).then(function (response) {\n if (response.success) {\n window.location.reload();\n } else {\n (0, _ezq.ezAlert)({\n title: \"Error!\",\n body: \"Logo uploading failed!\",\n button: \"Okay\"\n });\n }\n });\n });\n}\n\nfunction switchUserMode(event) {\n event.preventDefault();\n\n if (confirm(\"Are you sure you'd like to switch user modes?\\n\\nAll user submissions, awards, unlocks, and tracking will be deleted!\")) {\n var formData = new FormData();\n formData.append(\"submissions\", true);\n formData.append(\"nonce\", _CTFd[\"default\"].config.csrfNonce);\n fetch(_CTFd[\"default\"].config.urlRoot + \"/admin/reset\", {\n method: \"POST\",\n credentials: \"same-origin\",\n body: formData\n }); // Bind `this` so that we can reuse the updateConfigs function\n\n var binded = updateConfigs.bind(this);\n binded(event);\n }\n}\n\nfunction removeLogo() {\n (0, _ezq.ezQuery)({\n title: \"Remove logo\",\n body: \"Are you sure you'd like to remove the CTF logo?\",\n success: function success() {\n var params = {\n value: null\n };\n\n _CTFd[\"default\"].api.patch_config({\n configKey: \"ctf_logo\"\n }, params).then(function (_response) {\n window.location.reload();\n });\n }\n });\n}\n\nfunction smallIconUpload(event) {\n event.preventDefault();\n var form = event.target;\n\n _helpers[\"default\"].files.upload(form, {}, function (response) {\n var f = response.data[0];\n var params = {\n value: f.location\n };\n\n _CTFd[\"default\"].fetch(\"/api/v1/configs/ctf_small_icon\", {\n method: \"PATCH\",\n body: JSON.stringify(params)\n }).then(function (response) {\n return response.json();\n }).then(function (response) {\n if (response.success) {\n window.location.reload();\n } else {\n (0, _ezq.ezAlert)({\n title: \"Error!\",\n body: \"Icon uploading failed!\",\n button: \"Okay\"\n });\n }\n });\n });\n}\n\nfunction removeSmallIcon() {\n (0, _ezq.ezQuery)({\n title: \"Remove logo\",\n body: \"Are you sure you'd like to remove the small site icon?\",\n success: function success() {\n var params = {\n value: null\n };\n\n _CTFd[\"default\"].api.patch_config({\n configKey: \"ctf_small_icon\"\n }, params).then(function (_response) {\n window.location.reload();\n });\n }\n });\n}\n\nfunction importConfig(event) {\n event.preventDefault();\n var import_file = document.getElementById(\"import-file\").files[0];\n var form_data = new FormData();\n form_data.append(\"backup\", import_file);\n form_data.append(\"nonce\", _CTFd[\"default\"].config.csrfNonce);\n var pg = (0, _ezq.ezProgressBar)({\n width: 0,\n title: \"Upload Progress\"\n });\n\n _jquery[\"default\"].ajax({\n url: _CTFd[\"default\"].config.urlRoot + \"/admin/import\",\n type: \"POST\",\n data: form_data,\n processData: false,\n contentType: false,\n statusCode: {\n 500: function _(resp) {\n alert(resp.responseText);\n }\n },\n xhr: function xhr() {\n var xhr = _jquery[\"default\"].ajaxSettings.xhr();\n\n xhr.upload.onprogress = function (e) {\n if (e.lengthComputable) {\n var width = e.loaded / e.total * 100;\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: width\n });\n }\n };\n\n return xhr;\n },\n success: function success(_data) {\n pg = (0, _ezq.ezProgressBar)({\n target: pg,\n width: 100\n });\n setTimeout(function () {\n pg.modal(\"hide\");\n }, 500);\n setTimeout(function () {\n window.location.reload();\n }, 700);\n }\n });\n}\n\nfunction exportConfig(event) {\n event.preventDefault();\n window.location.href = (0, _jquery[\"default\"])(this).attr(\"href\");\n}\n\nfunction insertTimezones(target) {\n var current = (0, _jquery[\"default\"])(\"