From 1f8bc5b490d0e032529bb12e3d606cfa6d22d5f4 Mon Sep 17 00:00:00 2001 From: Nicolas Dorier Date: Fri, 10 Feb 2023 11:43:46 +0900 Subject: [PATCH] Add ability to migrate from MySQL and SQLite with EF (#4614) --- .../Configuration/DataDirectories.cs | 9 + BTCPayServer.Data/ApplicationDbContext.cs | 6 + .../Configuration/BTCPayServerOptions.cs | 12 +- BTCPayServer/Hosting/BTCPayServerServices.cs | 9 +- BTCPayServer/Hosting/MigrationStartupTask.cs | 9 +- .../Hosting/ToPostgresMigrationStartupTask.cs | 254 ++++++++++++++++++ docs/db-migration.md | 46 ++++ 7 files changed, 335 insertions(+), 10 deletions(-) create mode 100644 BTCPayServer/Hosting/ToPostgresMigrationStartupTask.cs create mode 100644 docs/db-migration.md diff --git a/BTCPayServer.Abstractions/Configuration/DataDirectories.cs b/BTCPayServer.Abstractions/Configuration/DataDirectories.cs index 3841a0979..b5aaf893f 100644 --- a/BTCPayServer.Abstractions/Configuration/DataDirectories.cs +++ b/BTCPayServer.Abstractions/Configuration/DataDirectories.cs @@ -1,3 +1,5 @@ +using System.IO; + namespace BTCPayServer.Configuration { public class DataDirectories @@ -7,5 +9,12 @@ namespace BTCPayServer.Configuration public string TempStorageDir { get; set; } public string StorageDir { get; set; } public string TempDir { get; set; } + + public string ToDatadirFullPath(string path) + { + if (Path.IsPathRooted(path)) + return path; + return Path.Combine(DataDir, path); + } } } diff --git a/BTCPayServer.Data/ApplicationDbContext.cs b/BTCPayServer.Data/ApplicationDbContext.cs index af188a067..cbc6224bf 100644 --- a/BTCPayServer.Data/ApplicationDbContext.cs +++ b/BTCPayServer.Data/ApplicationDbContext.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Threading.Tasks; using BTCPayServer.Data.Data; using Microsoft.AspNetCore.Identity.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; @@ -31,6 +32,11 @@ namespace BTCPayServer.Data _designTime = designTime; } + public async Task GetMigrationState() + { + return (await Settings.FromSqlRaw("SELECT \"Id\", \"Value\" FROM \"Settings\" WHERE \"Id\"='MigrationData'").AsNoTracking().FirstOrDefaultAsync())?.Value; + } + public DbSet AddressInvoices { get; set; } public DbSet ApiKeys { get; set; } public DbSet Apps { get; set; } diff --git a/BTCPayServer/Configuration/BTCPayServerOptions.cs b/BTCPayServer/Configuration/BTCPayServerOptions.cs index cc0b0b6cf..363c75b52 100644 --- a/BTCPayServer/Configuration/BTCPayServerOptions.cs +++ b/BTCPayServer/Configuration/BTCPayServerOptions.cs @@ -65,10 +65,14 @@ namespace BTCPayServer.Configuration if (conf.GetOrDefault("launchsettings", false) && NetworkType != ChainName.Regtest) throw new ConfigException($"You need to run BTCPayServer with the run.sh or run.ps1 script"); - if (conf.GetOrDefault("SQLITEFILE", null) != null) - Logs.Configuration.LogWarning("SQLITE backend support is deprecated and will be soon out of support"); - if (conf.GetOrDefault("MYSQL", null) != null) - Logs.Configuration.LogWarning("MYSQL backend support is deprecated and will be soon out of support"); + if (conf.GetOrDefault("POSTGRES", null) == null) + { + + if (conf.GetOrDefault("SQLITEFILE", null) != null) + Logs.Configuration.LogWarning("SQLITE backend support is out of support. Please migrate to Postgres by following the following instructions https://github.com/btcpayserver/btcpayserver/blob/master/docs/db-migration.md"); + if (conf.GetOrDefault("MYSQL", null) != null) + Logs.Configuration.LogWarning("MYSQL backend support is out of support. Please migrate to Postgres by following the following instructions (https://github.com/btcpayserver/btcpayserver/blob/master/docs/db-migration.md)"); + } DockerDeployment = conf.GetOrDefault("dockerdeployment", true); TorrcFile = conf.GetOrDefault("torrcfile", null); TorServices = conf.GetOrDefault("torservices", null) diff --git a/BTCPayServer/Hosting/BTCPayServerServices.cs b/BTCPayServer/Hosting/BTCPayServerServices.cs index ecf601f02..ee37e4528 100644 --- a/BTCPayServer/Hosting/BTCPayServerServices.cs +++ b/BTCPayServer/Hosting/BTCPayServerServices.cs @@ -128,7 +128,10 @@ namespace BTCPayServer.Hosting services.TryAddSingleton(o => o.GetRequiredService>().Value); // Don't move this StartupTask, we depend on it being right here + if (configuration["POSTGRES"] != null && (configuration["SQLITEFILE"] != null || configuration["MYSQL"] != null)) + services.AddStartupTask(); services.AddStartupTask(); + // AddSettingsAccessor(services); AddSettingsAccessor(services); @@ -174,12 +177,8 @@ namespace BTCPayServer.Hosting } else if (!string.IsNullOrEmpty(sqliteFileName)) { - var connStr = "Data Source=" + (Path.IsPathRooted(sqliteFileName) - ? sqliteFileName - : Path.Combine(datadirs.Value.DataDir, sqliteFileName)); - options.DatabaseType = DatabaseType.Sqlite; - options.ConnectionString = connStr; + options.ConnectionString = "Data Source=" + datadirs.Value.ToDatadirFullPath(sqliteFileName); } else { diff --git a/BTCPayServer/Hosting/MigrationStartupTask.cs b/BTCPayServer/Hosting/MigrationStartupTask.cs index 55c31a2ed..107b0c6b6 100644 --- a/BTCPayServer/Hosting/MigrationStartupTask.cs +++ b/BTCPayServer/Hosting/MigrationStartupTask.cs @@ -686,9 +686,16 @@ WHERE cte.""Id""=p.""Id"" retry: try { - await _DBContextFactory.CreateContext().Database.MigrateAsync(); + var db = _DBContextFactory.CreateContext(); + await db.Database.MigrateAsync(); + if (db.Database.IsNpgsql()) + { + if (await db.GetMigrationState() == "pending") + throw new ConfigException("This database hasn't been completely migrated, please retry migration by setting the BTCPAY_SQLITEFILE or BTCPAY_MYSQL setting on top of BTCPAY_POSTGRES"); + } } // Starting up + catch (ConfigException) { throw; } catch when (!cts.Token.IsCancellationRequested) { try diff --git a/BTCPayServer/Hosting/ToPostgresMigrationStartupTask.cs b/BTCPayServer/Hosting/ToPostgresMigrationStartupTask.cs new file mode 100644 index 000000000..bda33d183 --- /dev/null +++ b/BTCPayServer/Hosting/ToPostgresMigrationStartupTask.cs @@ -0,0 +1,254 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Amazon.Runtime.Internal.Util; +using AngleSharp.Text; +using BTCPayServer.Abstractions.Contracts; +using BTCPayServer.Configuration; +using BTCPayServer.Data; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Identity; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Conventions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MySqlConnector; +using NBXplorer; +using Newtonsoft.Json.Linq; +using Npgsql; + +namespace BTCPayServer.Hosting +{ + static class TopologySort + { + public static IEnumerable OrderByTopology(this IEnumerable tables) + { + var comparer = Comparer.Create((a, b) => a.Name.CompareTo(b.Name)); + return OrderByTopology( + tables, + t => + { + if (t.Name == "Invoices") + return t.ForeignKeyConstraints.Select(f => f.PrincipalTable.Name).Where(f => f != "Refunds"); + else + return t.ForeignKeyConstraints.Select(f => f.PrincipalTable.Name); + }, + t => t.Name, + t => t, + comparer); + } + public static IEnumerable OrderByTopology( + this IEnumerable values, + Func> dependsOn, + Func getKey, + Func getValue, + IComparer? solveTies = null) where T : notnull + { + var v = values.ToList(); + return TopologicalSort(v, dependsOn, getKey, getValue, solveTies); + } + + static List TopologicalSort(this IReadOnlyCollection nodes, + Func> dependsOn, + Func getKey, + Func getValue, + IComparer? solveTies = null) where T : notnull + { + if (nodes.Count == 0) + return new List(); + if (getKey == null) + throw new ArgumentNullException(nameof(getKey)); + if (getValue == null) + throw new ArgumentNullException(nameof(getValue)); + solveTies = solveTies ?? Comparer.Default; + List result = new List(nodes.Count); + HashSet allKeys = new HashSet(nodes.Count); + var noDependencies = new SortedDictionary>(solveTies); + + foreach (var node in nodes) + allKeys.Add(getKey(node)); + var dependenciesByValues = nodes.ToDictionary(node => node, + node => new HashSet(dependsOn(node).Where(n => allKeys.Contains(n)))); + foreach (var e in dependenciesByValues.Where(x => x.Value.Count == 0)) + { + noDependencies.Add(e.Key, e.Value); + } + if (noDependencies.Count == 0) + { + throw new InvalidOperationException("Impossible to topologically sort a cyclic graph"); + } + while (noDependencies.Count > 0) + { + var nodep = noDependencies.First(); + noDependencies.Remove(nodep.Key); + dependenciesByValues.Remove(nodep.Key); + + var elemKey = getKey(nodep.Key); + result.Add(getValue(nodep.Key)); + foreach (var selem in dependenciesByValues) + { + if (selem.Value.Remove(elemKey) && selem.Value.Count == 0) + noDependencies.Add(selem.Key, selem.Value); + } + } + if (dependenciesByValues.Count != 0) + { + throw new InvalidOperationException("Impossible to topologically sort a cyclic graph"); + } + return result; + } + } + public class ToPostgresMigrationStartupTask : IStartupTask + { + + public ToPostgresMigrationStartupTask( + IConfiguration configuration, + IOptions datadirs, + ILogger logger, + IWebHostEnvironment environment, + ApplicationDbContextFactory dbContextFactory) + { + Configuration = configuration; + Datadirs = datadirs; + Logger = logger; + Environment = environment; + DbContextFactory = dbContextFactory; + } + + public IConfiguration Configuration { get; } + public IOptions Datadirs { get; } + public ILogger Logger { get; } + public IWebHostEnvironment Environment { get; } + public ApplicationDbContextFactory DbContextFactory { get; } + public bool HasError { get; private set; } + + public async Task ExecuteAsync(CancellationToken cancellationToken = default) + { + var p = Configuration.GetOrDefault("POSTGRES", null); + var sqlite = Configuration.GetOrDefault("SQLITEFILE", null); + var mysql = Configuration.GetOrDefault("MYSQL", null); + + string migratingFrom; + ApplicationDbContext otherContext; + if (string.IsNullOrEmpty(p)) + { + return; + } + else if (!string.IsNullOrEmpty(sqlite)) + { + migratingFrom = "SQLite"; + sqlite = Datadirs.Value.ToDatadirFullPath(sqlite); + if (!File.Exists(sqlite)) + return; + otherContext = new ApplicationDbContext(new DbContextOptionsBuilder().UseSqlite("Data Source=" + sqlite, o => o.CommandTimeout(60 * 60 * 10)).Options); + } + else if (!string.IsNullOrEmpty(mysql)) + { + migratingFrom = "MySQL"; + otherContext = new ApplicationDbContext(new DbContextOptionsBuilder().UseMySql(mysql, ServerVersion.AutoDetect(mysql), o => o.CommandTimeout(60 * 60 * 10)).Options); + try + { + await otherContext.Settings.FirstOrDefaultAsync(); + } + catch (MySqlException ex) when (ex.SqlState == "42000") // DB doesn't exists + { + return; + } + } + else + { + return; + } + if (await otherContext.Settings.FirstOrDefaultAsync() == null) + return; + { + var postgres = new NpgsqlConnectionStringBuilder(p); + using var postgresContext = new ApplicationDbContext(new DbContextOptionsBuilder().UseNpgsql(p, o => o.CommandTimeout(60 * 60 * 10)).Options); + string? state; + try + { + state = await GetMigrationState(postgresContext); + if (state == "complete") + return; + if (state == null) + throw new ConfigException("This postgres database isn't created during a migration. Please use an empty database for postgres when migrating. If it's not a migration, remove --sqlitefile or --mysql settings."); + } + catch (NpgsqlException ex) when (ex.SqlState == PostgresErrorCodes.InvalidCatalogName) // DB doesn't exists + { + await postgresContext.Database.MigrateAsync(); + state = "pending"; + await SetMigrationState(postgresContext, migratingFrom, "pending"); + } + + Logger.LogInformation($"Migrating from {migratingFrom} to Postgres..."); + if (state == "pending") + { + Logger.LogInformation($"There is a unfinished migration in postgres... dropping all tables"); + foreach (var t in postgresContext.Model.GetRelationalModel().Tables.OrderByTopology()) + { + await postgresContext.Database.ExecuteSqlRawAsync($"DROP TABLE IF EXISTS \"{t.Name}\" CASCADE"); + } + await postgresContext.Database.ExecuteSqlRawAsync($"DROP TABLE IF EXISTS \"__EFMigrationsHistory\" CASCADE"); + await postgresContext.Database.MigrateAsync(); + } + else + { + throw new ConfigException("This database isn't created during a migration. Please use an empty database for postgres when migrating."); + } + await otherContext.Database.MigrateAsync(); + + await SetMigrationState(postgresContext, migratingFrom, "pending"); + + foreach (var t in postgresContext.Model.GetRelationalModel().Tables.OrderByTopology()) + { + var typeMapping = t.EntityTypeMappings.Single(); + var query = (IQueryable)otherContext.GetType().GetMethod("Set", new Type[0])!.MakeGenericMethod(typeMapping.EntityType.ClrType).Invoke(otherContext, null)!; + Logger.LogInformation($"Migrating table: " + t.Name); + var rows = await query.ToListAsync(); + foreach (var row in rows) + { + // There is as circular deps between invoice and refund. + if (row is InvoiceData id) + id.CurrentRefundId = null; + postgresContext.Entry(row).State = EntityState.Added; + } + await postgresContext.SaveChangesAsync(); + postgresContext.ChangeTracker.Clear(); + } + foreach (var invoice in otherContext.Invoices.AsNoTracking().Where(i => i.CurrentRefundId != null)) + { + postgresContext.Entry(invoice).State = EntityState.Modified; + } + await postgresContext.SaveChangesAsync(); + postgresContext.ChangeTracker.Clear(); + await SetMigrationState(postgresContext, migratingFrom, "complete"); + } + otherContext.Dispose(); + SqliteConnection.ClearAllPools(); + MySqlConnection.ClearAllPools(); + + Logger.LogInformation($"Migration to postgres from {migratingFrom} successful"); + } + + + private static async Task GetMigrationState(ApplicationDbContext postgresContext) + { + return (await postgresContext.Settings.FromSqlRaw("SELECT \"Id\", \"Value\" FROM \"Settings\" WHERE \"Id\"='MigrationData'").AsNoTracking().FirstOrDefaultAsync())?.Value; + } + private static async Task SetMigrationState(ApplicationDbContext postgresContext, string migratingFrom, string state) + { + await postgresContext.Database.ExecuteSqlRawAsync( + "INSERT INTO \"Settings\" VALUES ('MigrationData', @p0::JSONB) ON CONFLICT (\"Id\") DO UPDATE SET \"Value\"=@p0::JSONB", + new[] { $"{{ \"from\": \"{migratingFrom}\", \"state\": \"{state}\" }}" }); + } + } +} diff --git a/docs/db-migration.md b/docs/db-migration.md new file mode 100644 index 000000000..bc912504c --- /dev/null +++ b/docs/db-migration.md @@ -0,0 +1,46 @@ + +# Migration from SQLite and MySQL to Postgres + +## Introduction + +This document is intended for BTCPay Server integrators such as Raspiblitz, Umbrel, Embassy OS or anybody running BTCPay Server on SQLite or MySql. + +If you are a user of an integrated solution, please contact the integrator directly and provide them with the link to this document. + +BTCPay Server has for long time supported three different backends: +1. Postgres +2. SQLite +3. MySql + +While most of our users are using the Postgres backend, maintaining supports for all those databases has been very challenging, and Postgres is the only one part of our test suite. + +As a result, we regret to inform you that we decided to stop the support of MySql and SQLite. + +We understand that dropping support might be painful for users and integrators of our product, and we will do our best to provide a migration path. + +Please keep us informed if you experience any issues while migrating on [our community chat](https://chat.btcpayserver.org). + +## Procedure + +In order to successfully migrate, you will need to run BTCPay Server `1.7.8 or older`. + +As a reminder there are three settings controlling the choice of backend of BTCPay Server which can be controller by command line, environment variable or configuration settings. + +| Command line argument | Environment variable | +|---|---| +| --postgres | BTCPAY_POSTGRES="..." | +| --mysql | BTCPAY_MYSQL="..." | +| --sqlitefile | BTCPAY_SQLITEFILE="blah.db" | + +If you are currently using `mysql` or `sqlitefile`, and you wish to migrate to postgres, you simply need to add the command line argument `--postgres` or the environment variable `BTCPAY_POSTGRES` pointing to a fresh postgres database. + +From `1.7.8`, BTCPay Server will interprete this and attempt to copy the data from mysql and sqlite into the new postgres database. + +Note that once the migration is complete, the old `mysql` and `sqlite` settings will simply be ignored. + +If the migration fails, you can revert the `postgres` setting you added, so the next restart will run on the old unsupported database. You can retry a migration by adding the `postgres` setting again. + +## Known issues + +* The migration script isn't very optimized, and will attempt to load every table in memory. If your `sqlite` or `mysql` database is too big, you may experience an Out Of Memory issue. If that happen to you, please contact us. +* There are no migration for plugin's data. \ No newline at end of file