diff --git a/CTFd/themes/admin/static/js/keys/keys-utils.js b/CTFd/themes/admin/static/js/keys/keys-utils.js
index 5803393b..e0f03311 100644
--- a/CTFd/themes/admin/static/js/keys/keys-utils.js
+++ b/CTFd/themes/admin/static/js/keys/keys-utils.js
@@ -111,6 +111,8 @@ $(document).ready(function () {
                 var option = "<option value='{0}'>{1}</option>".format(key, data[key]);
                 $("#create-keys-select").append(option);
             }
+            $("#create-keys-entry-div").empty();
+            $("#create-keys-button-div").hide();
             $("#create-keys").modal();
         });
     });