From 7c92ce771fe7ad6f8e6ca49da91e133daf1db03d Mon Sep 17 00:00:00 2001
From: Nicolas Dorier <nicolas.dorier@gmail.com>
Date: Tue, 10 Sep 2024 17:34:02 +0900
Subject: [PATCH] Reindex Invoices table if corrupt, fix migration timeout
 (#6207)

---
 .../HostedServices/BlobMigratorHostedService.cs | 17 +++++++++++++++--
 .../InvoiceBlobMigratorHostedService.cs         | 14 +++++++++++++-
 BTCPayServer/Hosting/MigrationStartupTask.cs    |  6 +++---
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/BTCPayServer/HostedServices/BlobMigratorHostedService.cs b/BTCPayServer/HostedServices/BlobMigratorHostedService.cs
index ae170d8bf..e6221a014 100644
--- a/BTCPayServer/HostedServices/BlobMigratorHostedService.cs
+++ b/BTCPayServer/HostedServices/BlobMigratorHostedService.cs
@@ -67,14 +67,23 @@ public abstract class BlobMigratorHostedService<TEntity> : IHostedService
 retry:
             List<TEntity> entities;
             DateTimeOffset progress;
-            await using (var ctx = ApplicationDbContextFactory.CreateContext())
+            await using (var ctx = ApplicationDbContextFactory.CreateContext(o => o.CommandTimeout((int)TimeSpan.FromDays(1.0).TotalSeconds)))
             {
                 var query = GetQuery(ctx, settings?.Progress).Take(batchSize);
                 entities = await query.ToListAsync(cancellationToken);
                 if (entities.Count == 0)
                 {
+                    var count = await GetQuery(ctx, null).CountAsync(cancellationToken);
+                    if (count != 0)
+                    {
+                        settings = new Settings() { Progress = null };
+                        Logs.LogWarning("Corruption detected, reindexing the table...");
+                        await Reindex(ctx, cancellationToken);
+                        goto retry;
+                    }
                     await SettingsRepository.UpdateSetting<Settings>(new Settings() { Complete = true }, SettingsKey);
                     Logs.LogInformation("Migration completed");
+                    await PostMigrationCleanup(ctx, cancellationToken);
                     return;
                 }
 
@@ -84,7 +93,7 @@ retry:
                     await ctx.SaveChangesAsync();
                     batchSize = BatchSize;
                 }
-                catch (DbUpdateConcurrencyException)
+                catch (Exception ex) when (ex is DbUpdateConcurrencyException or TimeoutException or OperationCanceledException)
                 {
                     batchSize /= 2;
                     batchSize = Math.Max(1, batchSize);
@@ -95,6 +104,10 @@ retry:
             await SettingsRepository.UpdateSetting<Settings>(settings, SettingsKey);
         }
     }
+
+    protected abstract Task PostMigrationCleanup(ApplicationDbContext ctx, CancellationToken cancellationToken);
+
+    protected abstract Task Reindex(ApplicationDbContext ctx, CancellationToken cancellationToken);
     protected abstract IQueryable<TEntity> GetQuery(ApplicationDbContext ctx, DateTimeOffset? progress);
     protected abstract DateTimeOffset ProcessEntities(ApplicationDbContext ctx, List<TEntity> entities);
     public async Task ResetMigration()
diff --git a/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs b/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs
index 2b4abfc78..7bcf1a694 100644
--- a/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs
+++ b/BTCPayServer/HostedServices/InvoiceBlobMigratorHostedService.cs
@@ -9,6 +9,7 @@ using AngleSharp.Dom;
 using BTCPayServer.Abstractions.Contracts;
 using BTCPayServer.Data;
 using BTCPayServer.Services.Invoices;
+using Dapper;
 using Google.Apis.Logging;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Hosting;
@@ -42,9 +43,13 @@ public class InvoiceBlobMigratorHostedService : BlobMigratorHostedService<Invoic
             ctx.Invoices.Include(o => o.Payments).Where(i => i.Currency == null);
         return query.OrderByDescending(i => i.Created);
     }
+    protected override Task Reindex(ApplicationDbContext ctx, CancellationToken cancellationToken)
+    {
+		return ctx.Database.GetDbConnection().ExecuteAsync(new("REINDEX INDEX \"IX_Invoices_Created\";REINDEX INDEX \"PK_Invoices\";", cancellationToken: cancellationToken));
+    }
     protected override DateTimeOffset ProcessEntities(ApplicationDbContext ctx, List<InvoiceData> invoices)
     {
-        // Those clean up the JSON blobs, and mark entities as modified
+        // Those clean up the JSON blobs
         foreach (var inv in invoices)
         {
             var blob = inv.GetBlob();
@@ -70,4 +75,11 @@ public class InvoiceBlobMigratorHostedService : BlobMigratorHostedService<Invoic
         }
         return invoices[^1].Created;
     }
+
+    protected override async Task PostMigrationCleanup(ApplicationDbContext ctx, CancellationToken cancellationToken)
+    {
+		// If this one never run it's not big deal...
+		await ctx.Database.GetDbConnection().ExecuteAsync(new("VACUUM (ANALYZE) \"Invoices\"", cancellationToken: cancellationToken));
+		Logs.LogInformation("Post migration VACUUM successfull");
+	}
 }
diff --git a/BTCPayServer/Hosting/MigrationStartupTask.cs b/BTCPayServer/Hosting/MigrationStartupTask.cs
index d9c8479ae..27201ba8b 100644
--- a/BTCPayServer/Hosting/MigrationStartupTask.cs
+++ b/BTCPayServer/Hosting/MigrationStartupTask.cs
@@ -868,15 +868,15 @@ WHERE cte.""Id""=p.""Id""
 
         private async Task Migrate(CancellationToken cancellationToken)
         {
-            int cancellationTimeout = 60 * 60 * 24;
-            using (CancellationTokenSource timeout = new CancellationTokenSource(cancellationTimeout))
+            TimeSpan cancellationTimeout = TimeSpan.FromDays(1.0);
+            using (CancellationTokenSource timeout = new CancellationTokenSource((int)cancellationTimeout.TotalMilliseconds))
             using (CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken))
             {
 retry:
                 try
                 {
                     _logger.LogInformation("Running the migration scripts...");
-                    var db = _DBContextFactory.CreateContext(o => o.CommandTimeout(cancellationTimeout + 1));
+                    var db = _DBContextFactory.CreateContext(o => o.CommandTimeout(((int)cancellationTimeout.TotalSeconds) + 1));
                     await db.Database.MigrateAsync(timeout.Token);
                     _logger.LogInformation("All migration scripts ran successfully");
                 }