From a3a9361ba5397a5ab158fd997748a6b084b98480 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Mon, 13 Apr 2020 15:17:28 +0900 Subject: [PATCH 1/2] Persist the Delayed Broadcaster --- .../Data/ApplicationDbContext.cs | 2 + BTCPayServer.Data/Data/PlannedTransaction.cs | 15 ++++ .../20200413052418_PlannedTransactions.cs | 34 ++++++++ .../ApplicationDbContextModelSnapshot.cs | 17 ++++ BTCPayServer.Tests/PayJoinTests.cs | 28 ++++++ .../Services/DelayedTransactionBroadcaster.cs | 86 +++++++++++++------ 6 files changed, 157 insertions(+), 25 deletions(-) create mode 100644 BTCPayServer.Data/Data/PlannedTransaction.cs create mode 100644 BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs diff --git a/BTCPayServer.Data/Data/ApplicationDbContext.cs b/BTCPayServer.Data/Data/ApplicationDbContext.cs index e1029a900..ac7d5d87c 100644 --- a/BTCPayServer.Data/Data/ApplicationDbContext.cs +++ b/BTCPayServer.Data/Data/ApplicationDbContext.cs @@ -35,6 +35,8 @@ namespace BTCPayServer.Data get; set; } + public DbSet PlannedTransactions { get; set; } + public DbSet Apps { get; set; diff --git a/BTCPayServer.Data/Data/PlannedTransaction.cs b/BTCPayServer.Data/Data/PlannedTransaction.cs new file mode 100644 index 000000000..f658df60c --- /dev/null +++ b/BTCPayServer.Data/Data/PlannedTransaction.cs @@ -0,0 +1,15 @@ +using System; +using System.ComponentModel.DataAnnotations; + +namespace BTCPayServer.Data +{ + public class PlannedTransaction + { + [Key] + [MaxLength(100)] + // Id in the format [cryptocode]-[txid] + public string Id { get; set; } + public DateTimeOffset BroadcastAt { get; set; } + public byte[] Blob { get; set; } + } +} diff --git a/BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs b/BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs new file mode 100644 index 000000000..3492c1f6c --- /dev/null +++ b/BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs @@ -0,0 +1,34 @@ +using System; +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace BTCPayServer.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20200413052418_PlannedTransactions")] + public partial class PlannedTransactions : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "PlannedTransactions", + columns: table => new + { + Id = table.Column(maxLength: 100, nullable: false), + BroadcastAt = table.Column(nullable: false), + Blob = table.Column(nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_PlannedTransactions", x => x.Id); + }); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "PlannedTransactions"); + } + } +} diff --git a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs index f27375070..a775870a1 100644 --- a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -356,6 +356,23 @@ namespace BTCPayServer.Migrations b.ToTable("PendingInvoices"); }); + modelBuilder.Entity("BTCPayServer.Data.PlannedTransaction", b => + { + b.Property("Id") + .HasColumnType("TEXT") + .HasMaxLength(100); + + b.Property("Blob") + .HasColumnType("BLOB"); + + b.Property("BroadcastAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("PlannedTransactions"); + }); + modelBuilder.Entity("BTCPayServer.Data.RefundAddressesData", b => { b.Property("Id") diff --git a/BTCPayServer.Tests/PayJoinTests.cs b/BTCPayServer.Tests/PayJoinTests.cs index caa5d815c..57c831b7a 100644 --- a/BTCPayServer.Tests/PayJoinTests.cs +++ b/BTCPayServer.Tests/PayJoinTests.cs @@ -23,6 +23,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; using NBitcoin; +using NBitcoin.Altcoins; using NBitcoin.Payment; using NBitpayClient; using OpenQA.Selenium; @@ -41,6 +42,33 @@ namespace BTCPayServer.Tests Logs.LogProvider = new XUnitLogProvider(helper); } + [Fact] + [Trait("Integration", "Integration")] + public async Task CanUseTheDelayedBroadcaster() + { + using (var tester = ServerTester.Create()) + { + await tester.StartAsync(); + var network = tester.NetworkProvider.GetNetwork("BTC"); + var broadcaster = tester.PayTester.GetService(); + await broadcaster.Schedule(DateTimeOffset.UtcNow + TimeSpan.FromDays(500), RandomTransaction(network), network); + await broadcaster.Schedule(DateTimeOffset.UtcNow - TimeSpan.FromDays(5), RandomTransaction(network), network); + broadcaster.Disable(); + Assert.Equal(0, await broadcaster.ProcessAll()); + broadcaster.Enable(); + Assert.Equal(1, await broadcaster.ProcessAll()); + Assert.Equal(0, await broadcaster.ProcessAll()); + } + } + + private Transaction RandomTransaction(BTCPayNetwork network) + { + var tx = network.NBitcoinNetwork.CreateTransaction(); + tx.Inputs.Add(new OutPoint(RandomUtils.GetUInt256(), 0), Script.Empty); + tx.Outputs.Add(Money.Coins(1.0m), new Key().ScriptPubKey); + return tx; + } + [Fact] [Trait("Integration", "Integration")] public async Task CanOnlyUseCorrectAddressFormatsForPayjoin() diff --git a/BTCPayServer/Services/DelayedTransactionBroadcaster.cs b/BTCPayServer/Services/DelayedTransactionBroadcaster.cs index 5e6e4a50d..4207ec2b4 100644 --- a/BTCPayServer/Services/DelayedTransactionBroadcaster.cs +++ b/BTCPayServer/Services/DelayedTransactionBroadcaster.cs @@ -7,7 +7,9 @@ using System.Threading.Tasks; using NBXplorer; using System.Threading.Channels; using System.Threading; +using BTCPayServer.Data; using BTCPayServer.Logging; +using Microsoft.EntityFrameworkCore; namespace BTCPayServer.Services { @@ -15,52 +17,62 @@ namespace BTCPayServer.Services { class Record { - public DateTimeOffset Recorded; + public string Id; public DateTimeOffset BroadcastTime; public Transaction Transaction; public BTCPayNetwork Network; } - Channel _Records = Channel.CreateUnbounded(); - private readonly ExplorerClientProvider _explorerClientProvider; - public DelayedTransactionBroadcaster(ExplorerClientProvider explorerClientProvider) + private readonly BTCPayNetworkProvider _networkProvider; + private readonly ExplorerClientProvider _explorerClientProvider; + private readonly ApplicationDbContextFactory _dbContextFactory; + + public DelayedTransactionBroadcaster( + BTCPayNetworkProvider networkProvider, + ExplorerClientProvider explorerClientProvider, + Data.ApplicationDbContextFactory dbContextFactory) { if (explorerClientProvider == null) throw new ArgumentNullException(nameof(explorerClientProvider)); + _networkProvider = networkProvider; _explorerClientProvider = explorerClientProvider; + _dbContextFactory = dbContextFactory; } - public Task Schedule(DateTimeOffset broadcastTime, Transaction transaction, BTCPayNetwork network) + public async Task Schedule(DateTimeOffset broadcastTime, Transaction transaction, BTCPayNetwork network) { if (transaction == null) throw new ArgumentNullException(nameof(transaction)); if (network == null) throw new ArgumentNullException(nameof(network)); - var now = DateTimeOffset.UtcNow; - var record = new Record() + using (var db = _dbContextFactory.CreateContext()) { - Recorded = now, - BroadcastTime = broadcastTime, - Transaction = transaction, - Network = network - }; - _Records.Writer.TryWrite(record); - // TODO: persist - return Task.CompletedTask; + db.PlannedTransactions.Add(new PlannedTransaction() + { + Id = $"{network.CryptoCode}-{transaction.GetHash()}", + BroadcastAt = broadcastTime, + Blob = transaction.ToBytes() + }); + await db.SaveChangesAsync(); + } } - public async Task ProcessAll(CancellationToken cancellationToken = default) + public async Task ProcessAll(CancellationToken cancellationToken = default) { if (disabled) - return; - var now = DateTimeOffset.UtcNow; - List rescheduled = new List(); + return 0; List scheduled = new List(); - List broadcasted = new List(); - while (_Records.Reader.TryRead(out var r)) + using (var db = _dbContextFactory.CreateContext()) { - (r.BroadcastTime > now ? rescheduled : scheduled).Add(r); + scheduled = (await db.PlannedTransactions + .ToListAsync()).Select(ToRecord) + .Where(r => r != null) + // Client side filtering because entity framework is retarded. + .Where(r => r.BroadcastTime < DateTimeOffset.UtcNow).ToList(); } + + List rescheduled = new List(); + List broadcasted = new List(); var broadcasts = scheduled.Select(async (record) => { @@ -89,11 +101,30 @@ namespace BTCPayServer.Services var needReschedule = await broadcasts[i]; (needReschedule ? rescheduled : broadcasted).Add(scheduled[i]); } - foreach (var record in rescheduled) + + using (var db = _dbContextFactory.CreateContext()) { - _Records.Writer.TryWrite(record); + foreach (Record record in broadcasted) + { + db.PlannedTransactions.Remove(new PlannedTransaction() {Id = record.Id}); + } + return await db.SaveChangesAsync(); } - // TODO: Remove everything in broadcasted from DB + } + + private Record ToRecord(PlannedTransaction plannedTransaction) + { + var s = plannedTransaction.Id.Split('-'); + var network = _networkProvider.GetNetwork(s[0]) as BTCPayNetwork; + if (network is null) + return null; + return new Record() + { + Id = plannedTransaction.Id, + Network = network, + Transaction = Transaction.Load(plannedTransaction.Blob, network.NBitcoinNetwork), + BroadcastTime = plannedTransaction.BroadcastAt + }; } private bool disabled = false; @@ -101,5 +132,10 @@ namespace BTCPayServer.Services { disabled = true; } + + public void Enable() + { + disabled = false; + } } } From c73c34dfaaff49eb3e8e827526b07ee4897f7f02 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Mon, 13 Apr 2020 15:43:25 +0900 Subject: [PATCH 2/2] Persisting locked input and outpoints --- .../Data/ApplicationDbContext.cs | 1 + BTCPayServer.Data/Data/PayjoinLock.cs | 16 +++++ .../20200413052418_PlannedTransactions.cs | 12 ++++ .../ApplicationDbContextModelSnapshot.cs | 11 +++ BTCPayServer.Tests/PayJoinTests.cs | 36 +++++++++- .../Payments/PayJoin/PayJoinRepository.cs | 71 +++++++++++++------ .../Services/DelayedTransactionBroadcaster.cs | 8 ++- 7 files changed, 133 insertions(+), 22 deletions(-) create mode 100644 BTCPayServer.Data/Data/PayjoinLock.cs diff --git a/BTCPayServer.Data/Data/ApplicationDbContext.cs b/BTCPayServer.Data/Data/ApplicationDbContext.cs index ac7d5d87c..50cca746d 100644 --- a/BTCPayServer.Data/Data/ApplicationDbContext.cs +++ b/BTCPayServer.Data/Data/ApplicationDbContext.cs @@ -36,6 +36,7 @@ namespace BTCPayServer.Data } public DbSet PlannedTransactions { get; set; } + public DbSet PayjoinLocks { get; set; } public DbSet Apps { diff --git a/BTCPayServer.Data/Data/PayjoinLock.cs b/BTCPayServer.Data/Data/PayjoinLock.cs new file mode 100644 index 000000000..dd8f11588 --- /dev/null +++ b/BTCPayServer.Data/Data/PayjoinLock.cs @@ -0,0 +1,16 @@ +using System.ComponentModel.DataAnnotations; + +namespace BTCPayServer.Data +{ + /// + /// We represent the locks of the PayjoinRepository + /// with this table. (Both, our utxo we locked as part of a payjoin + /// and the utxo of the payer which were used to pay us) + /// + public class PayjoinLock + { + [Key] + [MaxLength(100)] + public string Id { get; set; } + } +} diff --git a/BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs b/BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs index 3492c1f6c..a011925c7 100644 --- a/BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs +++ b/BTCPayServer.Data/Migrations/20200413052418_PlannedTransactions.cs @@ -23,10 +23,22 @@ namespace BTCPayServer.Migrations { table.PrimaryKey("PK_PlannedTransactions", x => x.Id); }); + migrationBuilder.CreateTable( + name: "PayjoinLocks", + columns: table => new + { + Id = table.Column(maxLength: 100, nullable: false), + }, + constraints: table => + { + table.PrimaryKey("PK_PayjoinLocks", x => x.Id); + }); } protected override void Down(MigrationBuilder migrationBuilder) { + migrationBuilder.DropTable( + name: "PayjoinLocks"); migrationBuilder.DropTable( name: "PlannedTransactions"); } diff --git a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs index a775870a1..47add10fe 100644 --- a/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/BTCPayServer.Data/Migrations/ApplicationDbContextModelSnapshot.cs @@ -297,6 +297,17 @@ namespace BTCPayServer.Migrations b.ToTable("PairingCodes"); }); + modelBuilder.Entity("BTCPayServer.Data.PayjoinLock", b => + { + b.Property("Id") + .HasColumnType("TEXT") + .HasMaxLength(100); + + b.HasKey("Id"); + + b.ToTable("PayjoinLocks"); + }); + modelBuilder.Entity("BTCPayServer.Data.PaymentData", b => { b.Property("Id") diff --git a/BTCPayServer.Tests/PayJoinTests.cs b/BTCPayServer.Tests/PayJoinTests.cs index 57c831b7a..deb0b9ff3 100644 --- a/BTCPayServer.Tests/PayJoinTests.cs +++ b/BTCPayServer.Tests/PayJoinTests.cs @@ -52,7 +52,10 @@ namespace BTCPayServer.Tests var network = tester.NetworkProvider.GetNetwork("BTC"); var broadcaster = tester.PayTester.GetService(); await broadcaster.Schedule(DateTimeOffset.UtcNow + TimeSpan.FromDays(500), RandomTransaction(network), network); - await broadcaster.Schedule(DateTimeOffset.UtcNow - TimeSpan.FromDays(5), RandomTransaction(network), network); + var tx = RandomTransaction(network); + await broadcaster.Schedule(DateTimeOffset.UtcNow - TimeSpan.FromDays(5), tx, network); + // twice on same tx should be noop + await broadcaster.Schedule(DateTimeOffset.UtcNow - TimeSpan.FromDays(5), tx, network); broadcaster.Disable(); Assert.Equal(0, await broadcaster.ProcessAll()); broadcaster.Enable(); @@ -60,6 +63,32 @@ namespace BTCPayServer.Tests Assert.Equal(0, await broadcaster.ProcessAll()); } } + [Fact] + [Trait("Integration", "Integration")] + public async Task CanUsePayjoinRepository() + { + using (var tester = ServerTester.Create()) + { + await tester.StartAsync(); + var network = tester.NetworkProvider.GetNetwork("BTC"); + var repo = tester.PayTester.GetService(); + var outpoint = RandomOutpoint(); + + // Should not be locked + Assert.False(await repo.TryUnlock(outpoint)); + + // Can lock input + Assert.True(await repo.TryLockInputs(new [] { outpoint })); + // Can't twice + Assert.False(await repo.TryLockInputs(new [] { outpoint })); + Assert.False(await repo.TryUnlock(outpoint)); + + // Lock and unlock outpoint utxo + Assert.True(await repo.TryLock(outpoint)); + Assert.True(await repo.TryUnlock(outpoint)); + Assert.False(await repo.TryUnlock(outpoint)); + } + } private Transaction RandomTransaction(BTCPayNetwork network) { @@ -69,6 +98,11 @@ namespace BTCPayServer.Tests return tx; } + private OutPoint RandomOutpoint() + { + return new OutPoint(RandomUtils.GetUInt256(), 0); + } + [Fact] [Trait("Integration", "Integration")] public async Task CanOnlyUseCorrectAddressFormatsForPayjoin() diff --git a/BTCPayServer/Payments/PayJoin/PayJoinRepository.cs b/BTCPayServer/Payments/PayJoin/PayJoinRepository.cs index f67e2a0fa..87998db13 100644 --- a/BTCPayServer/Payments/PayJoin/PayJoinRepository.cs +++ b/BTCPayServer/Payments/PayJoin/PayJoinRepository.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using BTCPayServer.Data; +using Microsoft.EntityFrameworkCore; using NBitcoin; using NBXplorer.Models; @@ -9,40 +11,69 @@ namespace BTCPayServer.Payments.PayJoin { public class PayJoinRepository { - HashSet _Outpoints = new HashSet(); - HashSet _LockedInputs = new HashSet(); - public Task TryLock(OutPoint outpoint) + private readonly ApplicationDbContextFactory _dbContextFactory; + + public PayJoinRepository(ApplicationDbContextFactory dbContextFactory) { - lock (_Outpoints) + _dbContextFactory = dbContextFactory; + } + public async Task TryLock(OutPoint outpoint) + { + using var ctx = _dbContextFactory.CreateContext(); + ctx.PayjoinLocks.Add(new PayjoinLock() { - return Task.FromResult(_Outpoints.Add(outpoint)); + Id = outpoint.ToString() + }); + try + { + return await ctx.SaveChangesAsync() == 1; + } + catch (DbUpdateException e) + { + return false; } } - public Task TryUnlock(params OutPoint[] outPoints) + public async Task TryUnlock(params OutPoint[] outPoints) { - if (outPoints.Length == 0) - return Task.FromResult(true); - lock (_Outpoints) + using var ctx = _dbContextFactory.CreateContext(); + foreach (OutPoint outPoint in outPoints) { - bool r = true; - foreach (var outpoint in outPoints) + ctx.PayjoinLocks.Remove(new PayjoinLock() { - r &= _Outpoints.Remove(outpoint); - } - return Task.FromResult(r); + Id = outPoint.ToString() + }); + } + try + { + return await ctx.SaveChangesAsync() == outPoints.Length; + } + catch (DbUpdateException e) + { + return false; } } - public Task TryLockInputs(OutPoint[] outPoint) + public async Task TryLockInputs(OutPoint[] outPoints) { - lock (_LockedInputs) + using var ctx = _dbContextFactory.CreateContext(); + foreach (OutPoint outPoint in outPoints) { - foreach (var o in outPoint) - if (!_LockedInputs.Add(o)) - return Task.FromResult(false); + ctx.PayjoinLocks.Add(new PayjoinLock() + { + // Random flag so it does not lock same id + // as the lock utxo + Id = "K-" + outPoint.ToString() + }); + } + try + { + return await ctx.SaveChangesAsync() == outPoints.Length; + } + catch (DbUpdateException e) + { + return false; } - return Task.FromResult(true); } } } diff --git a/BTCPayServer/Services/DelayedTransactionBroadcaster.cs b/BTCPayServer/Services/DelayedTransactionBroadcaster.cs index 4207ec2b4..10fd63497 100644 --- a/BTCPayServer/Services/DelayedTransactionBroadcaster.cs +++ b/BTCPayServer/Services/DelayedTransactionBroadcaster.cs @@ -53,7 +53,13 @@ namespace BTCPayServer.Services BroadcastAt = broadcastTime, Blob = transaction.ToBytes() }); - await db.SaveChangesAsync(); + try + { + await db.SaveChangesAsync(); + } + catch (DbUpdateException) + { + } } }