Refactor the InvoiceAddresses table (#6232)

BTCPayServer.Common/BTCPayNetwork.cs

@@ -133,7 +133,7 @@ namespace BTCPayServer
 
         public string GetTrackedDestination(Script scriptPubKey)
         {
-            return scriptPubKey.Hash.ToString() + "#" + CryptoCode.ToUpperInvariant();
+            return scriptPubKey.Hash.ToString();
         }
     }
 

BTCPayServer.Data/ApplicationDbContext.cs

@@ -84,7 +84,6 @@ namespace BTCPayServer.Data
             PaymentRequestData.OnModelCreating(builder, Database);
             PaymentData.OnModelCreating(builder, Database);
             PayoutData.OnModelCreating(builder, Database);
-            PendingInvoiceData.OnModelCreating(builder);
             //PlannedTransaction.OnModelCreating(builder);
             PullPaymentData.OnModelCreating(builder, Database);
             RefundData.OnModelCreating(builder);

BTCPayServer.Data/Data/AddressInvoiceData.cs

@@ -9,6 +9,7 @@ namespace BTCPayServer.Data
         public string Address { get; set; }
         public InvoiceData InvoiceData { get; set; }
         public string InvoiceDataId { get; set; }
+        public string PaymentMethodId { get; set; }
 
 
         internal static void OnModelCreating(ModelBuilder builder)
@@ -18,7 +19,7 @@ namespace BTCPayServer.Data
                    .WithMany(i => i.AddressInvoices).OnDelete(DeleteBehavior.Cascade);
             builder.Entity<AddressInvoiceData>()
 #pragma warning disable CS0618
-                .HasKey(o => o.Address);
+                .HasKey(o => new { o.PaymentMethodId, o.Address });
 #pragma warning restore CS0618
         }
     }

BTCPayServer.Data/Data/InvoiceData.cs

@@ -26,7 +26,6 @@ namespace BTCPayServer.Data
         public string ExceptionStatus { get; set; }
         public List<AddressInvoiceData> AddressInvoices { get; set; }
         public bool Archived { get; set; }
-        public List<PendingInvoiceData> PendingInvoices { get; set; }
         public List<InvoiceSearchData> InvoiceSearchData { get; set; }
         public List<RefundData> Refunds { get; set; }
 

BTCPayServer.Data/Data/PendingInvoiceData.cs

@@ -1,18 +0,0 @@
-using Microsoft.EntityFrameworkCore;
-
-namespace BTCPayServer.Data
-{
-    public class PendingInvoiceData
-    {
-        public string Id { get; set; }
-        public InvoiceData InvoiceData { get; set; }
-
-        internal static void OnModelCreating(ModelBuilder builder)
-        {
-            builder.Entity<PendingInvoiceData>()
-                .HasOne(o => o.InvoiceData)
-                .WithMany(o => o.PendingInvoices)
-                .HasForeignKey(o => o.Id).OnDelete(DeleteBehavior.Cascade);
-        }
-    }
-}

BTCPayServer.Data/Migrations/20240919085726_refactorinvoiceaddress.cs

@@ -0,0 +1,80 @@
+using BTCPayServer.Data;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace BTCPayServer.Migrations
+{
+    [DbContext(typeof(ApplicationDbContext))]
+    [Migration("20240919085726_refactorinvoiceaddress")]
+    public partial class refactorinvoiceaddress : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropPrimaryKey(
+                name: "PK_AddressInvoices",
+                table: "AddressInvoices");
+
+            migrationBuilder.AddColumn<string>(
+                name: "PaymentMethodId",
+                table: "AddressInvoices",
+                type: "text",
+                nullable: false,
+                defaultValue: "");
+
+            migrationBuilder.Sql("""
+                UPDATE "AddressInvoices"
+                SET 
+                "Address" = (string_to_array("Address", '#'))[1],
+                "PaymentMethodId" = CASE WHEN (string_to_array("Address", '#'))[2] IS NULL THEN 'BTC-CHAIN'
+                                         WHEN STRPOS((string_to_array("Address", '#'))[2], '_') = 0 THEN (string_to_array("Address", '#'))[2] || '-CHAIN'
+                                         WHEN STRPOS((string_to_array("Address", '#'))[2], '_MoneroLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_MoneroLike','-CHAIN')
+                                         WHEN STRPOS((string_to_array("Address", '#'))[2], '_ZcashLike') > 0 THEN replace((string_to_array("Address", '#'))[2],'_ZcashLike','-CHAIN')
+                                         ELSE '' END;
+
+                DELETE FROM "AddressInvoices" WHERE "PaymentMethodId" = '';
+                """);
+            migrationBuilder.AddPrimaryKey(
+                name: "PK_AddressInvoices",
+                table: "AddressInvoices",
+                columns: new[] { "PaymentMethodId", "Address" });
+            migrationBuilder.Sql("VACUUM (ANALYZE) \"AddressInvoices\";", true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropPrimaryKey(
+                name: "PK_AddressInvoices",
+                table: "AddressInvoices");
+
+            migrationBuilder.DropColumn(
+                name: "PaymentMethodId",
+                table: "AddressInvoices");
+
+            migrationBuilder.AddPrimaryKey(
+                name: "PK_AddressInvoices",
+                table: "AddressInvoices",
+                column: "Address");
+
+            migrationBuilder.CreateTable(
+                name: "PendingInvoices",
+                columns: table => new
+                {
+                    Id = table.Column<string>(type: "text", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_PendingInvoices", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_PendingInvoices_Invoices_Id",
+                        column: x => x.Id,
+                        principalTable: "Invoices",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+        }
+    }
+}

BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs

@@ -60,13 +60,16 @@ namespace BTCPayServer.Migrations
 
             modelBuilder.Entity("BTCPayServer.Data.AddressInvoiceData", b =>
                 {
+                    b.Property<string>("PaymentMethodId")
+                        .HasColumnType("text");
+
                     b.Property<string>("Address")
                         .HasColumnType("text");
 
                     b.Property<string>("InvoiceDataId")
                         .HasColumnType("text");
 
-                    b.HasKey("Address");
+                    b.HasKey("PaymentMethodId", "Address");
 
                     b.HasIndex("InvoiceDataId");
 
@@ -634,16 +637,6 @@ namespace BTCPayServer.Migrations
                     b.ToTable("PayoutProcessors");
                 });
 
-            modelBuilder.Entity("BTCPayServer.Data.PendingInvoiceData", b =>
-                {
-                    b.Property<string>("Id")
-                        .HasColumnType("text");
-
-                    b.HasKey("Id");
-
-                    b.ToTable("PendingInvoices");
-                });
-
             modelBuilder.Entity("BTCPayServer.Data.PlannedTransaction", b =>
                 {
                     b.Property<string>("Id")
@@ -1331,17 +1324,6 @@ namespace BTCPayServer.Migrations
                     b.Navigation("Store");
                 });
 
-            modelBuilder.Entity("BTCPayServer.Data.PendingInvoiceData", b =>
-                {
-                    b.HasOne("BTCPayServer.Data.InvoiceData", "InvoiceData")
-                        .WithMany("PendingInvoices")
-                        .HasForeignKey("Id")
-                        .OnDelete(DeleteBehavior.Cascade)
-                        .IsRequired();
-
-                    b.Navigation("InvoiceData");
-                });
-
             modelBuilder.Entity("BTCPayServer.Data.PullPaymentData", b =>
                 {
                     b.HasOne("BTCPayServer.Data.StoreData", "StoreData")
@@ -1572,8 +1554,6 @@ namespace BTCPayServer.Migrations
 
                     b.Navigation("Payments");
 
-                    b.Navigation("PendingInvoices");
-
                     b.Navigation("Refunds");
                 });
 

BTCPayServer.Tests/DatabaseTests.cs

@@ -18,6 +18,26 @@ namespace BTCPayServer.Tests
         {
         }
 
+        [Fact]
+        public async Task CanMigrateInvoiceAddresses()
+        {
+            var tester = CreateDBTester();
+            await tester.MigrateUntil("20240919085726_refactorinvoiceaddress");
+            using var ctx = tester.CreateContext();
+            var conn = ctx.Database.GetDbConnection();
+            await conn.ExecuteAsync("INSERT INTO \"Invoices\" (\"Id\", \"Created\") VALUES ('i', NOW())");
+            await conn.ExecuteAsync(
+                "INSERT INTO \"AddressInvoices\" VALUES ('aaa#BTC', 'i'),('bbb','i'),('ccc#BTC_LNU', 'i'),('ddd#XMR_MoneroLike', 'i'),('eee#ZEC_ZcashLike', 'i')");
+            await tester.ContinueMigration();
+            foreach (var v in new[] { ("aaa", "BTC-CHAIN"), ("bbb", "BTC-CHAIN"), ("ddd", "XMR-CHAIN") , ("eee", "ZEC-CHAIN") })
+            {
+                var ok = await conn.ExecuteScalarAsync<bool>("SELECT 't'::BOOLEAN FROM \"AddressInvoices\" WHERE \"Address\"=@a AND \"PaymentMethodId\"=@b", new { a = v.Item1, b = v.Item2 });
+                Assert.True(ok);
+            }
+            var notok = await conn.ExecuteScalarAsync<bool>("SELECT 't'::BOOLEAN FROM \"AddressInvoices\" WHERE \"Address\"='ccc'");
+            Assert.False(notok);
+        }
+
         [Fact]
         public async Task CanMigratePayoutsAndPullPayments()
         {

BTCPayServer.Tests/TestUtils.cs

@@ -16,7 +16,7 @@ namespace BTCPayServer.Tests
     public static class TestUtils
     {
 #if DEBUG && !SHORT_TIMEOUT
-        public const int TestTimeout = 600_000;
+        public const int TestTimeout = 60_000;
 #else
         public const int TestTimeout = 90_000;
 #endif

BTCPayServer.Tests/UnitTest1.cs

@@ -2449,9 +2449,10 @@ namespace BTCPayServer.Tests
         private static bool IsMapped(Invoice invoice, ApplicationDbContext ctx)
         {
             var h = BitcoinAddress.Create(invoice.BitcoinAddress, Network.RegTest).ScriptPubKey.Hash.ToString();
+            var pmi = PaymentTypes.CHAIN.GetPaymentMethodId("BTC");
             return (ctx.AddressInvoices.Where(i => i.InvoiceDataId == invoice.Id).ToArrayAsync().GetAwaiter()
                     .GetResult())
-                .Where(i => i.GetAddress() == h).Any();
+                .Where(i => i.Address == h && i.PaymentMethodId == pmi.ToString()).Any();
         }
 
 

BTCPayServer/Controllers/UIInvoiceController.UI.cs

@@ -649,9 +649,7 @@ namespace BTCPayServer.Controllers
                     if (derivationScheme is null)
                         return NotSupported("This feature is only available to BTC wallets");
                     var btc = PaymentTypes.CHAIN.GetPaymentMethodId("BTC");
-                    var bumpableAddresses = (await GetAddresses(selectedItems))
+                    var bumpableAddresses = await GetAddresses(btc, selectedItems);
-                                            .Where(p => p.GetPaymentMethodId() == btc)
-                                            .Select(p => p.GetAddress()).ToHashSet();
                     var utxos = await explorer.GetUTXOsAsync(derivationScheme);
                     var bumpableUTXOs = utxos.GetUnspentUTXOs().Where(u => u.Confirmations == 0 && bumpableAddresses.Contains(u.ScriptPubKey.Hash.ToString())).ToArray();
                     var parameters = new MultiValueDictionary<string, string>();
@@ -673,10 +671,10 @@ namespace BTCPayServer.Controllers
             return RedirectToAction(nameof(ListInvoices), new { storeId });
         }
 
-        private async Task<AddressInvoiceData[]> GetAddresses(string[] selectedItems)
+        private async Task<HashSet<string>> GetAddresses(PaymentMethodId paymentMethodId, string[] selectedItems)
         {
             using var ctx = _dbContextFactory.CreateContext();
-            return await ctx.AddressInvoices.Where(i => selectedItems.Contains(i.InvoiceDataId)).ToArrayAsync();
+            return new HashSet<string>(await ctx.AddressInvoices.Where(i => i.PaymentMethodId == paymentMethodId.ToString() && selectedItems.Contains(i.InvoiceDataId)).Select(i => i.Address).ToArrayAsync());
         }
 
         [HttpGet("i/{invoiceId}")]

BTCPayServer/Data/AddressInvoiceDataExtensions.cs

@@ -1,31 +0,0 @@
-using System;
-using BTCPayServer.Payments;
-
-namespace BTCPayServer.Data
-{
-    public static class AddressInvoiceDataExtensions
-    {
-#pragma warning disable CS0618
-        public static string GetAddress(this AddressInvoiceData addressInvoiceData)
-        {
-            if (addressInvoiceData.Address == null)
-                return null;
-            var index = addressInvoiceData.Address.LastIndexOf("#", StringComparison.InvariantCulture);
-            if (index == -1)
-                return addressInvoiceData.Address;
-            return addressInvoiceData.Address.Substring(0, index);
-        }
-        public static PaymentMethodId GetPaymentMethodId(this AddressInvoiceData addressInvoiceData)
-        {
-            if (addressInvoiceData.Address == null)
-                return null;
-            var index = addressInvoiceData.Address.LastIndexOf("#", StringComparison.InvariantCulture);
-            // Legacy AddressInvoiceData does not have the paymentMethodId attached to the Address
-            if (index == -1)
-                return PaymentMethodId.Parse("BTC");
-            /////////////////////////
-            return PaymentMethodId.TryParse(addressInvoiceData.Address.Substring(index + 1));
-        }
-#pragma warning restore CS0618
-    }
-}

BTCPayServer/Data/InvoiceDataExtensions.cs

@@ -73,7 +73,7 @@ namespace BTCPayServer.Data
             entity.Status = state.Status;
             if (invoiceData.AddressInvoices != null)
             {
-                entity.AvailableAddressHashes = invoiceData.AddressInvoices.Select(a => a.GetAddress() + a.GetPaymentMethodId()).ToHashSet();
+                entity.Addresses = invoiceData.AddressInvoices.Select(a => (PaymentMethodId.Parse(a.PaymentMethodId), a.Address)).ToHashSet();
             }
             if (invoiceData.Refunds != null)
             {

BTCPayServer/Payments/Bitcoin/NBXplorerListener.cs

@@ -144,6 +144,7 @@ namespace BTCPayServer.Payments.Bitcoin
                     Logs.PayServer.LogInformation($"{network.CryptoCode}: {paymentCount} payments happened while offline");
 
                     Logs.PayServer.LogInformation($"Connected to WebSocket of NBXplorer ({network.CryptoCode})");
+                    var pmi = PaymentTypes.CHAIN.GetPaymentMethodId(network.CryptoCode);
                     while (!_Cts.IsCancellationRequested)
                     {
                         var newEvent = await session.NextEventAsync(_Cts.Token).ConfigureAwait(false);
@@ -163,13 +164,11 @@ namespace BTCPayServer.Payments.Bitcoin
                                     foreach (var output in validOutputs)
                                     {
                                         var key = network.GetTrackedDestination(output.Item1.ScriptPubKey);
-                                        var invoice = (await _InvoiceRepository.GetInvoicesFromAddresses(new[] { key }))
+                                        var invoice = await _InvoiceRepository.GetInvoiceFromAddress(pmi, key);
-                                            .FirstOrDefault();
                                         if (invoice != null)
                                         {
                                             var address = output.matchedOutput.Address ?? network.NBXplorerNetwork.CreateAddress(evt.DerivationStrategy,
                                                 output.Item1.KeyPath, output.Item1.ScriptPubKey);
-                                            var pmi = PaymentTypes.CHAIN.GetPaymentMethodId(network.CryptoCode);
                                             var handler = _handlers[pmi];
                                             var details = new BitcoinLikePaymentData(output.outPoint, evt.TransactionData.Transaction.RBF, output.matchedOutput.KeyPath);
 
@@ -198,7 +197,6 @@ namespace BTCPayServer.Payments.Bitcoin
                                                 await UpdatePaymentStates(wallet, invoice.Id);
                                             }
                                         }
-
                                     }
                                 }
 
@@ -406,8 +404,7 @@ namespace BTCPayServer.Payments.Bitcoin
                     coins = await wallet.GetUnspentCoins(strategy);
                     coinsPerDerivationStrategy.Add(strategy, coins);
                 }
-                coins = coins.Where(c => invoice.AvailableAddressHashes.Contains(c.ScriptPubKey.Hash.ToString() + cryptoId))
+                coins = coins.Where(c => invoice.Addresses.Contains((cryptoId, network.GetTrackedDestination(c.ScriptPubKey)))).ToArray();
-                             .ToArray();
                 foreach (var coin in coins.Where(c => !alreadyAccounted.Contains(c.OutPoint)))
                 {
                     var transaction = await wallet.GetTransactionAsync(coin.OutPoint.Hash);

BTCPayServer/Payments/IPaymentMethodHandler.cs

@@ -280,7 +280,7 @@ namespace BTCPayServer.Payments
         /// <summary>
         /// This string can be used to query AddressInvoice to find the invoiceId
         /// </summary>
-        public List<string> TrackedDestinations { get; } = new List<string>();
+        public List<string> TrackedDestinations { get; } = new();
 
         internal async Task BeforeFetchingRates()
         {

BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs

@@ -265,8 +265,8 @@ namespace BTCPayServer.Payments.PayJoin
                 if (walletReceiveMatch is null)
                 {
 
-                    var key = output.ScriptPubKey.Hash + "#" + network.CryptoCode.ToUpperInvariant();
+                    var key = network.GetTrackedDestination(output.ScriptPubKey);
-                    invoice = (await _invoiceRepository.GetInvoicesFromAddresses(new[] { key })).FirstOrDefault();
+                    invoice = await _invoiceRepository.GetInvoiceFromAddress(paymentMethodId, key);
                     if (invoice is null)
                         continue;
                     accountDerivation = _handlers.GetDerivationStrategy(invoice, network);

BTCPayServer/Services/Altcoins/Monero/Services/MoneroListener.cs

@@ -284,12 +284,9 @@ namespace BTCPayServer.Services.Altcoins.Monero.Services
             foreach (var destination in transfer.Transfers.GroupBy(destination => destination.Address))
             {
                 //find the invoice corresponding to this address, else skip
-                var address = destination.Key + "#" + paymentMethodId;
+                var invoice = await _invoiceRepository.GetInvoiceFromAddress(paymentMethodId, destination.Key);
-                var invoice = (await _invoiceRepository.GetInvoicesFromAddresses(new[] { address })).FirstOrDefault();
                 if (invoice == null)
-                {
                     continue;
-                }
 
                 var index = destination.First().SubaddrIndex;
 

BTCPayServer/Services/Altcoins/Zcash/Services/ZcashListener.cs

@@ -282,12 +282,9 @@ namespace BTCPayServer.Services.Altcoins.Zcash.Services
             foreach (var destination in transfer.Transfers.GroupBy(destination => destination.Address))
             {
                 //find the invoice corresponding to this address, else skip
-                var address = destination.Key + "#" + paymentMethodId;
+                var invoice = await _invoiceRepository.GetInvoiceFromAddress(paymentMethodId, destination.Key);
-                var invoice = (await _invoiceRepository.GetInvoicesFromAddresses(new[] { address })).FirstOrDefault();
                 if (invoice == null)
-                {
                     continue;
-                }
 
                 var index = destination.First().SubaddrIndex;
 

BTCPayServer/Services/Invoices/InvoiceEntity.cs

@@ -495,7 +495,7 @@ namespace BTCPayServer.Services.Invoices
         public DateTimeOffset MonitoringExpiration { get; set; }
 
         [JsonIgnore]
-        public HashSet<string> AvailableAddressHashes { get; set; }
+        public HashSet<(PaymentMethodId PaymentMethodId, string Address)> Addresses { get; set; }
         [JsonProperty]
         public bool ExtendedNotifications { get; set; }
 

BTCPayServer/Services/Invoices/InvoiceRepository.cs

@@ -67,29 +67,16 @@ namespace BTCPayServer.Services.Invoices
             };
         }
 
-        public async Task<IEnumerable<InvoiceEntity>> GetInvoicesFromAddresses(string[] addresses)
+        public async Task<InvoiceEntity> GetInvoiceFromAddress(PaymentMethodId paymentMethodId, string address)
         {
-            if (addresses.Length is 0)
-                return Array.Empty<InvoiceEntity>();
             using var db = _applicationDbContextFactory.CreateContext();
-            if (addresses.Length == 1)
+			var row = (await db.AddressInvoices
-            {
+			.Include(a => a.InvoiceData.Payments)
-                var address = addresses[0];
+				.Where(a => a.PaymentMethodId == paymentMethodId.ToString() && a.Address == address)
-                return (await db.AddressInvoices
+				.Select(a => a.InvoiceData)
-                .Include(a => a.InvoiceData.Payments)
+			.FirstOrDefaultAsync());
-                    .Where(a => a.Address == address)
+			return row is null ? null : ToEntity(row);
-                    .Select(a => a.InvoiceData)
+		}
-                .ToListAsync()).Select(ToEntity);
-            }
-            else
-            {
-                return (await db.AddressInvoices
-                    .Include(a => a.InvoiceData.Payments)
-                        .Where(a => addresses.Contains(a.Address))
-                        .Select(a => a.InvoiceData)
-                    .ToListAsync()).Select(ToEntity);
-            }
-        }
 
         public async Task<InvoiceEntity[]> GetInvoicesWithPendingPayments(PaymentMethodId paymentMethodId, bool includeAddresses = false)
         {
@@ -190,7 +177,8 @@ retry:
                         await context.AddressInvoices.AddAsync(new AddressInvoiceData()
                         {
                             InvoiceDataId = invoice.Id,
-                            Address = trackedDestination
+                            Address = trackedDestination,
+                            PaymentMethodId = ctx.Key.ToString()
                         });
                     }
                 }
@@ -311,7 +299,8 @@ retry:
                         await context.AddressInvoices.AddAsync(new AddressInvoiceData()
                         {
                             InvoiceDataId = invoiceId,
-                            Address = tracked
+                            Address = tracked,
+                            PaymentMethodId = paymentPromptContext.PaymentMethodId.ToString()
                         });
                     }
                     AddToTextSearch(context, invoice, prompt.Destination);