Ajaxify the wallet transaction list to avoid timeout (Fix #4987) (#5100)

* Ajaxify the wallet transaction list to avoid timeout (Fix #4987)

* Add cancellation to request to wallet transactions

* Fix tests

* Improve empty state

* Cleanups

---------

Co-authored-by: Dennis Reimann <mail@dennisreimann.de>

BTCPayServer.Tests/Extensions.cs

@@ -122,6 +122,13 @@ retry:
             driver.ExecuteJavaScript($"document.getElementById('{element}').{funcName}()");
         }
 
+        public static void WaitWalletTransactionsLoaded(this IWebDriver driver)
+        {
+            var wait = new WebDriverWait(driver, SeleniumTester.ImplicitWait);
+            wait.UntilJsIsReady();
+            wait.Until(d => d.WaitForElement(By.CssSelector("#WalletTransactions[data-loaded='true']")));
+        }
+
         public static IWebElement WaitForElement(this IWebDriver driver, By selector)
         {
             var wait = new WebDriverWait(driver, SeleniumTester.ImplicitWait);

BTCPayServer.Tests/SeleniumTests.cs

@@ -1441,7 +1441,7 @@ namespace BTCPayServer.Tests
             s.Driver.FindElement(By.Id("CancelWizard")).Click();
 
             // Check the label is applied to the tx
-
+            s.Driver.WaitWalletTransactionsLoaded();
             Assert.Equal("label2", s.Driver.FindElement(By.XPath("//*[@id=\"WalletTransactionsList\"]//*[contains(@class, 'transaction-label')]")).Text);
 
             //change the wallet and ensure old address is not there and generating a new one does not result in the prev one
@@ -1492,7 +1492,9 @@ namespace BTCPayServer.Tests
 
             // Check the tx sent earlier arrived
             s.Driver.FindElement(By.Id($"StoreNav-Wallet{cryptoCode}")).Click();
-            Assert.Contains(tx.ToString(), s.Driver.PageSource);
+            s.Driver.WaitWalletTransactionsLoaded();
+            s.Driver.FindElement(By.PartialLinkText(tx.ToString()));
+
             var walletTransactionUri = new Uri(s.Driver.Url);
 
             // Send to bob
@@ -1618,9 +1620,8 @@ namespace BTCPayServer.Tests
 
             // Transactions list is empty 
             s.Driver.FindElement(By.Id($"StoreNav-Wallet{cryptoCode}")).Click();
-            Assert.Contains("There are no transactions yet.", s.Driver.PageSource);
+            s.Driver.WaitWalletTransactionsLoaded();
-            s.Driver.AssertElementNotFound(By.Id("ExportDropdownToggle"));
+            Assert.Contains("There are no transactions yet", s.Driver.FindElement(By.Id("WalletTransactions")).Text);
-            s.Driver.AssertElementNotFound(By.Id("ActionsDropdownToggle"));
         }
 
         [Fact]

BTCPayServer.Tests/UnitTest1.cs

@@ -719,7 +719,7 @@ namespace BTCPayServer.Tests
                 btcDerivationScheme.GetDerivation(new KeyPath("0/90")).ScriptPubKey, Money.Coins(1.0m));
             tester.ExplorerNode.Generate(1);
             var transactions = Assert.IsType<ListTransactionsViewModel>(Assert
-                .IsType<ViewResult>(walletController.WalletTransactions(walletId).Result).Model);
+                .IsType<ViewResult>(walletController.WalletTransactions(walletId, loadTransactions: true).Result).Model);
             Assert.Empty(transactions.Transactions);
 
             Assert.IsType<RedirectToActionResult>(walletController.WalletRescan(walletId, rescan).Result);
@@ -748,7 +748,7 @@ namespace BTCPayServer.Tests
             Assert.NotNull(rescan.TimeOfScan);
             Assert.Equal(1, rescan.LastSuccess.Found);
             transactions = Assert.IsType<ListTransactionsViewModel>(Assert
-                .IsType<ViewResult>(walletController.WalletTransactions(walletId).Result).Model);
+                .IsType<ViewResult>(walletController.WalletTransactions(walletId, loadTransactions: true).Result).Model);
             var tx = Assert.Single(transactions.Transactions);
             Assert.Equal(tx.Id, txId.ToString());
 
@@ -763,7 +763,7 @@ namespace BTCPayServer.Tests
                 await walletController.ModifyTransaction(walletId, tx.Id, addcomment: "hello"));
 
             transactions = Assert.IsType<ListTransactionsViewModel>(Assert
-                .IsType<ViewResult>(walletController.WalletTransactions(walletId).Result).Model);
+                .IsType<ViewResult>(walletController.WalletTransactions(walletId, loadTransactions: true).Result).Model);
             tx = Assert.Single(transactions.Transactions);
 
             Assert.Equal("hello", tx.Comment);
@@ -775,7 +775,7 @@ namespace BTCPayServer.Tests
                 await walletController.ModifyTransaction(walletId, tx.Id, removelabel: "test2"));
 
             transactions = Assert.IsType<ListTransactionsViewModel>(Assert
-                .IsType<ViewResult>(walletController.WalletTransactions(walletId).Result).Model);
+                .IsType<ViewResult>(walletController.WalletTransactions(walletId, loadTransactions: true).Result).Model);
             tx = Assert.Single(transactions.Transactions);
 
             Assert.Equal("hello", tx.Comment);

BTCPayServer/Components/StoreRecentTransactions/StoreRecentTransactions.cs

@@ -58,7 +58,7 @@ public class StoreRecentTransactions : ViewComponent
         {
             var network = derivationSettings.Network;
             var wallet = _walletProvider.GetWallet(network);
-            var allTransactions = await wallet.FetchTransactionHistory(derivationSettings.AccountDerivation, 0, 5, TimeSpan.FromDays(31.0));
+            var allTransactions = await wallet.FetchTransactionHistory(derivationSettings.AccountDerivation, 0, 5, TimeSpan.FromDays(31.0), cancellationToken: this.HttpContext.RequestAborted);
             var walletTransactionsInfo = await _walletRepository.GetWalletTransactionsInfo(vm.WalletId, allTransactions.Select(t => t.TransactionId.ToString()).ToArray());
 
             transactions = allTransactions

BTCPayServer/Controllers/GreenField/GreenfieldStoreOnChainWalletsController.cs

@@ -182,7 +182,8 @@ namespace BTCPayServer.Controllers.Greenfield
             [FromQuery] TransactionStatus[]? statusFilter = null,
             [FromQuery] string? labelFilter = null,
             [FromQuery] int skip = 0,
-            [FromQuery] int limit = int.MaxValue
+            [FromQuery] int limit = int.MaxValue,
+            CancellationToken cancellationToken = default
         )
         {
             if (IsInvalidWalletRequest(cryptoCode, out var network,
@@ -197,7 +198,7 @@ namespace BTCPayServer.Controllers.Greenfield
             if (statusFilter?.Any() is true || !string.IsNullOrWhiteSpace(labelFilter))
                 preFiltering = false;
             var txs = await wallet.FetchTransactionHistory(derivationScheme.AccountDerivation, preFiltering ? skip : 0,
-                preFiltering ? limit : int.MaxValue);
+                preFiltering ? limit : int.MaxValue, cancellationToken: cancellationToken);
             if (!preFiltering)
             {
                 var filteredList = new List<TransactionHistoryLine>(txs.Count);

BTCPayServer/Controllers/UIWalletsController.cs

@@ -212,7 +212,9 @@ namespace BTCPayServer.Controllers
             WalletId walletId,
             string? labelFilter = null,
             int skip = 0,
-            int count = 50
+            int count = 50,
+            bool loadTransactions = false,
+            CancellationToken cancellationToken = default
         )
         {
             var paymentMethod = GetDerivationSchemeSettings(walletId);
@@ -223,25 +225,25 @@ namespace BTCPayServer.Controllers
 
             // We can't filter at the database level if we need to apply label filter
             var preFiltering = string.IsNullOrEmpty(labelFilter);
-            var transactions = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, preFiltering ? skip : null, preFiltering ? count : null);
-            var walletTransactionsInfo = await WalletRepository.GetWalletTransactionsInfo(walletId, transactions.Select(t => t.TransactionId.ToString()).ToArray());
             var model = new ListTransactionsViewModel { Skip = skip, Count = count };
             model.Labels.AddRange(
                 (await WalletRepository.GetWalletLabels(walletId))
                 .Select(c => (c.Label, c.Color, ColorPalette.Default.TextColor(c.Color))));
 
+            IList<TransactionHistoryLine>? transactions = null;
+            Dictionary<string, WalletTransactionInfo>? walletTransactionsInfo = null;
+            if (loadTransactions)
+            {
+                transactions = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, preFiltering ? skip : null, preFiltering ? count : null, cancellationToken: cancellationToken);
+                walletTransactionsInfo = await WalletRepository.GetWalletTransactionsInfo(walletId, transactions.Select(t => t.TransactionId.ToString()).ToArray());
+            }
+
             if (labelFilter != null)
             {
                 model.PaginationQuery = new Dictionary<string, object> { { "labelFilter", labelFilter } };
             }
-            if (transactions == null)
+            if (transactions == null || walletTransactionsInfo is null)
             {
-                TempData.SetStatusMessageModel(new StatusMessageModel()
-                {
-                    Severity = StatusMessageModel.StatusSeverity.Error,
-                    Message =
-                        "There was an error retrieving the transactions list. Is NBXplorer configured correctly?"
-                });
                 model.Transactions = new List<ListTransactionsViewModel.TransactionViewModel>();
             }
             else
@@ -1311,7 +1313,7 @@ namespace BTCPayServer.Controllers
         [HttpGet("{walletId}/export")]
         public async Task<IActionResult> Export(
             [ModelBinder(typeof(WalletIdModelBinder))] WalletId walletId,
-            string format, string? labelFilter = null)
+            string format, string? labelFilter = null, CancellationToken cancellationToken = default)
         {
             var paymentMethod = GetDerivationSchemeSettings(walletId);
             if (paymentMethod == null)
@@ -1319,7 +1321,7 @@ namespace BTCPayServer.Controllers
 
             var wallet = _walletProvider.GetWallet(paymentMethod.Network);
             var walletTransactionsInfoAsync = WalletRepository.GetWalletTransactionsInfo(walletId, (string[]?)null);
-            var input = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation);
+            var input = await wallet.FetchTransactionHistory(paymentMethod.AccountDerivation, cancellationToken: cancellationToken);
             var walletTransactionsInfo = await walletTransactionsInfoAsync;
             var export = new TransactionsExport(wallet, walletTransactionsInfo);
             var res = export.Process(input, format);

BTCPayServer/Services/Wallets/BTCPayWallet.cs

@@ -215,7 +215,7 @@ namespace BTCPayServer.Services.Wallets
             return await completionSource.Task;
         }
         List<TransactionInformation> dummy = new List<TransactionInformation>();
-        public async Task<IList<TransactionHistoryLine>> FetchTransactionHistory(DerivationStrategyBase derivationStrategyBase, int? skip = null, int? count = null, TimeSpan? interval = null)
+        public async Task<IList<TransactionHistoryLine>> FetchTransactionHistory(DerivationStrategyBase derivationStrategyBase, int? skip = null, int? count = null, TimeSpan? interval = null, CancellationToken cancellationToken = default)
         {
             // This is two paths:
             // * Sometimes we can ask the DB to do the filtering of rows: If that's the case, we should try to filter at the DB level directly as it is the most efficient.
@@ -243,18 +243,21 @@ namespace BTCPayServer.Services.Wallets
             else
             {
                 await using var ctx = await NbxplorerConnectionFactory.OpenConnection();
-                var rows = await ctx.QueryAsync<(string tx_id, DateTimeOffset seen_at, string blk_id, long? blk_height, long balance_change, string asset_id, long confs)>(
+                var cmd = new CommandDefinition(
-                    "SELECT r.tx_id, r.seen_at, t.blk_id, t.blk_height, r.balance_change, r.asset_id, COALESCE((SELECT height FROM get_tip('BTC')) - t.blk_height + 1, 0) AS confs " +
+                    commandText: "SELECT r.tx_id, r.seen_at, t.blk_id, t.blk_height, r.balance_change, r.asset_id, COALESCE((SELECT height FROM get_tip('BTC')) - t.blk_height + 1, 0) AS confs " +
                     "FROM get_wallets_recent(@wallet_id, @code, @interval, @count, @skip) r " +
                     "JOIN txs t USING (code, tx_id) " +
-                    "ORDER BY r.seen_at DESC", new
+                    "ORDER BY r.seen_at DESC",
+                    parameters: new
                     {
                         wallet_id = NBXplorer.Client.DBUtils.nbxv1_get_wallet_id(Network.CryptoCode, derivationStrategyBase.ToString()),
                         code = Network.CryptoCode,
                         count = count,
                         skip = skip,
                         interval = interval is TimeSpan t ? t : TimeSpan.FromDays(365 * 1000)
-                    });
+                    },
+                    cancellationToken: cancellationToken);
+                var rows = await ctx.QueryAsync<(string tx_id, DateTimeOffset seen_at, string blk_id, long? blk_height, long balance_change, string asset_id, long confs)>(cmd);
                 rows.TryGetNonEnumeratedCount(out int c);
                 var lines = new List<TransactionHistoryLine>(c);
                 foreach (var row in rows)

BTCPayServer/Views/UIWallets/WalletTransactions.cshtml

@@ -49,10 +49,10 @@
     <script src="~/modal/btcpay.js" asp-append-version="true" async></script>
     @* Custom Range Modal *@
     <script>
-        let observer = null;
-        let $loadMore = document.getElementById('LoadMore');
         const $actions = document.getElementById('ListActions');
+        const $transactions = document.getElementById('WalletTransactions');
         const $list = document.getElementById('WalletTransactionsList');
+        const $dropdowns = document.getElementById('Dropdowns');
         const $indicator = document.getElementById('LoadingIndicator');
         
         delegate('click', '#selectAllCheckbox', e => {
@@ -65,19 +65,15 @@
             window.scrollTo({ top: 0, behavior: 'smooth' });
         });
         
-        delegate('click', '#LoadMore', async () => {
-            $loadMore.setAttribute('disabled', 'disabled');
-            await loadMoreTransactions();
-        });
-        
         if ($actions && $actions.offsetTop - window.innerHeight > 0) {
             document.getElementById('GoToTop').classList.remove('d-none');
         }
         
         const count = @Safe.Json(Model.Count);
         const skipInitial = @Safe.Json(Model.Skip);
-        const loadMoreUrl = @Safe.Json(Url.Action("WalletTransactions", new { walletId, labelFilter, skip = Model.Skip, count = Model.Count }));
+		const loadMoreUrl = @Safe.Json(Url.Action("WalletTransactions", new { walletId, labelFilter, skip = Model.Skip, count = Model.Count, loadTransactions = true }));
-        let skip = @Safe.Json(Model.Skip);
+		// The next time we load transactions, skip will become 0
+        let skip = @Safe.Json(Model.Skip) - count;
         
         async function loadMoreTransactions() {
             $indicator.classList.remove('d-none');
@@ -93,38 +89,38 @@
             
             if (response.ok) {
                 const html = await response.text();
+                const responseEmpty = html.trim() === '';
                 $list.insertAdjacentHTML('beforeend', html);
                 skip = skipNext;
                 
-                if ($loadMore) {
+                if (responseEmpty) {
-                    // remove load more button
-                    $loadMore.remove();
-                    $loadMore = null;
-                    
-                    // switch to infinite scroll mode
-                    observer = new IntersectionObserver(async entries => {
-                        const { isIntersecting } = entries[0];
-                        if (isIntersecting) {
-                            await loadMoreTransactions();
-                        }
-                    }, { rootMargin: '128px' });
-                    
-                    // the actions div marks the end of the list table
-                    observer.observe($actions);
-                }
-                
-                if (html.trim() === '') {
                     // in case the response html was empty, remove the observer and stop loading
                     observer.unobserve($actions);
                 }
-            } else if ($loadMore) {
+                if (!$transactions.dataset.loaded) {
-                $loadMore.removeAttribute('disabled');
+                    $transactions.dataset.loaded = 'true';
+                    // replace table and dropdowns if initial response was empty
+                    if (responseEmpty) {
+                        $dropdowns.remove();
+                        $transactions.innerHTML = '<div class="text-secondary" data-loaded="true">There are no transactions yet.</div>';
+                    }
+                }
             }
             
             $indicator.classList.add('d-none');
             formatDateTimes(document.getElementById('switchTimeFormat').dataset.mode);
             initLabelManagers();
         }
+        
+		const observer = new IntersectionObserver(async entries => {
+            const { isIntersecting } = entries[0];
+            if (isIntersecting) {
+                await loadMoreTransactions();
+            }
+        }, { rootMargin: '128px' });
+        
+        // the actions div marks the end of the list table
+        observer.observe($actions);
     </script>
 }
 
@@ -147,80 +143,66 @@
     </div>
 }
 
-@if (Model.Transactions.Any())
+<div class="d-inline-flex align-items-center gap-3" id="Dropdowns">
-{
+    <div class="dropdown ms-auto" id="Actions">
-    <div class="d-inline-flex align-items-center gap-3" id="Dropdowns">
+        <button class="btn btn-secondary dropdown-toggle" type="button" id="ActionsDropdownToggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-        <div class="dropdown ms-auto" id="Actions">
+            Actions
-            <button class="btn btn-secondary dropdown-toggle" type="button" id="ActionsDropdownToggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-                Actions
-            </button>
-            <div class="dropdown-menu dropdown-menu-end" aria-labelledby="ActionsDropdownToggle">
-                <form id="WalletActions" method="post" asp-action="WalletActions" asp-route-walletId="@walletId">
-                    <button id="BumpFee" name="command" type="submit" class="dropdown-item" value="cpfp">Bump fee (CPFP)</button>
-                </form>
-            </div>
-        </div>
-        <div class="dropdown d-inline-flex align-items-center gap-3" id="Export">
-            <button class="btn btn-secondary dropdown-toggle" type="button" id="ExportDropdownToggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-                Export
-            </button>
-            <div class="dropdown-menu" aria-labelledby="ExportDropdownToggle">
-                <a asp-action="Export" asp-route-walletId="@walletId" asp-route-format="csv" asp-route-labelFilter="@labelFilter" class="dropdown-item export-link" target="_blank" id="ExportCSV">CSV</a>
-                <a asp-action="Export" asp-route-walletId="@walletId" asp-route-format="json" asp-route-labelFilter="@labelFilter" class="dropdown-item export-link" target="_blank" id="ExportJSON">JSON</a>
-                <a asp-action="Export" asp-route-walletId="@walletId" asp-route-format="bip329" asp-route-labelFilter="@labelFilter" class="dropdown-item export-link" target="_blank" id="ExportBIP329">Wallet Labels (BIP-329)</a>
-            </div>
-        </div>
-    </div>
-    <div style="clear:both"></div>
-
-    <div class="row">
-        <div class="col table-responsive-md" id="walletTable">
-            <table class="table table-hover">
-                <thead class="thead-inverse">
-                <tr>
-                    <th style="width:2rem;" class="only-for-js">
-                        <input id="selectAllCheckbox" type="checkbox" class="form-check-input" />
-                    </th>
-                    <th class="w-150px">
-                        <div class="d-flex align-items-center gap-1">
-                            Date
-                            <button type="button" class="btn btn-link p-0 fa fa-clock-o switch-time-format" title="Switch date format" id="switchTimeFormat"></button>
-                        </div>
-                    </th>
-                    <th class="text-start">Label</th>
-                    <th>Transaction Id</th>
-                    <th class="text-end">Amount</th>
-                    <th class="text-end" style="min-width:60px"></th>
-                </tr>
-                </thead>
-                <tbody id="WalletTransactionsList">
-                    <partial name="_WalletTransactionsList" model="Model" />
-                </tbody>
-            </table>
-        </div>
-    </div>
-    <noscript>
-        <vc:pager view-model="Model"/>
-    </noscript>
-    
-    <div class="text-center only-for-js d-none" id="LoadingIndicator">
-        <div class="spinner-border spinner-border-sm text-secondary ms-2" role="status">
-            <span class="visually-hidden">Loading...</span>
-        </div>
-    </div>
-    <div class="d-flex flex-wrap align-items-center justify-content-center gap-3 mb-5 only-for-js" id="ListActions">
-        <button type="button" class="btn btn-secondary d-flex align-items-center" id="LoadMore">
-            Load more
         </button>
-        <button type="button" class="btn btn-secondary d-none" id="GoToTop">Go to top</button>
+        <div class="dropdown-menu dropdown-menu-end" aria-labelledby="ActionsDropdownToggle">
+            <form id="WalletActions" method="post" asp-action="WalletActions" asp-route-walletId="@walletId">
+                <button id="BumpFee" name="command" type="submit" class="dropdown-item" value="cpfp">Bump fee (CPFP)</button>
+            </form>
+        </div>
     </div>
-}
+    <div class="dropdown d-inline-flex align-items-center gap-3" id="Export">
-else
+        <button class="btn btn-secondary dropdown-toggle" type="button" id="ExportDropdownToggle" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-{
+            Export
-    <p class="text-secondary mt-3">
+        </button>
-        There are no transactions @(string.IsNullOrEmpty(labelFilter) ? "yet" : $"labeled with \"{labelFilter}\"").
+        <div class="dropdown-menu" aria-labelledby="ExportDropdownToggle">
-    </p>
+            <a asp-action="Export" asp-route-walletId="@walletId" asp-route-format="csv" asp-route-labelFilter="@labelFilter" class="dropdown-item export-link" target="_blank" id="ExportCSV">CSV</a>
-}
+            <a asp-action="Export" asp-route-walletId="@walletId" asp-route-format="json" asp-route-labelFilter="@labelFilter" class="dropdown-item export-link" target="_blank" id="ExportJSON">JSON</a>
+            <a asp-action="Export" asp-route-walletId="@walletId" asp-route-format="bip329" asp-route-labelFilter="@labelFilter" class="dropdown-item export-link" target="_blank" id="ExportBIP329">Wallet Labels (BIP-329)</a>
+        </div>
+    </div>
+</div>
+<div style="clear:both"></div>
+
+<div id="WalletTransactions" class="table-responsive-md">
+    <table class="table table-hover">
+        <thead class="thead-inverse">
+        <tr>
+            <th style="width:2rem;" class="only-for-js">
+                <input id="selectAllCheckbox" type="checkbox" class="form-check-input" />
+            </th>
+            <th class="w-150px">
+                <div class="d-flex align-items-center gap-1">
+                    Date
+                    <button type="button" class="btn btn-link p-0 fa fa-clock-o switch-time-format" title="Switch date format" id="switchTimeFormat"></button>
+                </div>
+            </th>
+            <th class="text-start">Label</th>
+            <th>Transaction Id</th>
+            <th class="text-end">Amount</th>
+            <th class="text-end" style="min-width:60px"></th>
+        </tr>
+        </thead>
+        <tbody id="WalletTransactionsList">
+            <partial name="_WalletTransactionsList" model="Model" />
+        </tbody>
+    </table>
+</div>
+<noscript>
+    <vc:pager view-model="Model"/>
+</noscript>
+    
+<div class="text-center only-for-js d-none" id="LoadingIndicator">
+    <div class="spinner-border spinner-border-sm text-secondary ms-2" role="status">
+        <span class="visually-hidden">Loading...</span>
+    </div>
+</div>
+<div class="d-flex flex-wrap align-items-center justify-content-center gap-3 mb-5 only-for-js" id="ListActions">
+    <button type="button" class="btn btn-secondary d-none" id="GoToTop">Go to top</button>
+</div>
 
 <p class="mt-4 mb-0">
     If BTCPay Server shows you an invalid balance, <a asp-action="WalletRescan" asp-route-walletId="@Context.GetRouteValue("walletId")">rescan your wallet</a>.