Do not save cache of rates in the database (#6978)

We were previously saving the rates in the database in a JSONB blob
column. However, the volume of data ise consequential enough for
provoking timeouts during update.

Due to how postgres works, this also create bloat in the database that
isn't cleaned immediately.

This PR fixes this issue by saving the cache in files instead.

BTCPayServer/Extensions.cs

@@ -38,6 +38,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using NBitcoin;
 using NBitcoin.Payment;
 using NBitcoin.RPC;
@@ -448,7 +449,7 @@ namespace BTCPayServer
         public static IServiceCollection AddScheduledTask<T>(this IServiceCollection services, TimeSpan every)
             where T : class, IPeriodicTask
         {
-            services.AddSingleton<T>();
+            services.TryAddSingleton<T>();
             services.AddTransient<ScheduledTask>(o => new ScheduledTask(typeof(T), every));
             return services;
         }

BTCPayServer/HostedServices/RatesHostedService.cs

@@ -1,47 +1,34 @@
+#nullable enable
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using BTCPayServer.Logging;
+using BTCPayServer.Configuration;
-using BTCPayServer.Services;
 using BTCPayServer.Services.Rates;
-using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
 using NBitcoin;
 using Newtonsoft.Json;
 
 namespace BTCPayServer.HostedServices
 {
-    public class RatesHostedService : BaseAsyncService
+    public class RatesHostedService(
+        IOptions<DataDirectories> dataDirectories,
+        RateProviderFactory rateProviderFactory) : IHostedService, IPeriodicTask
     {
         public class ExchangeRatesCache
         {
             [JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))]
             public DateTimeOffset Created { get; set; }
-            public List<BackgroundFetcherState> States { get; set; }
+            public List<BackgroundFetcherState>? States { get; set; }
             public override string ToString()
             {
                 return "";
             }
         }
-        private readonly SettingsRepository _SettingsRepository;
-        readonly RateProviderFactory _RateProviderFactory;
-
-        public RatesHostedService(SettingsRepository repo,
-                                  RateProviderFactory rateProviderFactory,
-                                  Logs logs) : base(logs)
-        {
-            this._SettingsRepository = repo;
-            _RateProviderFactory = rateProviderFactory;
-        }
-
-        internal override Task[] InitializeTasks()
-        {
-            return new Task[]
-            {
-                CreateLoopTask(RefreshRates)
-            };
-        }
 
         bool IsStillUsed(BackgroundFetcherRateProvider fetcher)
         {
@@ -51,7 +38,7 @@ namespace BTCPayServer.HostedServices
 
         IEnumerable<(string ExchangeName, BackgroundFetcherRateProvider Fetcher)> GetStillUsedProviders()
         {
-            foreach (var kv in _RateProviderFactory.Providers)
+            foreach (var kv in rateProviderFactory.Providers)
             {
                 if (kv.Value is BackgroundFetcherRateProvider fetcher && IsStillUsed(fetcher))
                 {
@@ -59,33 +46,23 @@ namespace BTCPayServer.HostedServices
                 }
             }
         }
-        async Task RefreshRates()
+        public async Task Do(CancellationToken cancellationToken)
         {
             var usedProviders = GetStillUsedProviders().ToArray();
             if (usedProviders.Length == 0)
-            {
-                await Task.Delay(TimeSpan.FromSeconds(30), CancellationToken);
                 return;
-            }
+            using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-            using (var timeout = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken))
-            {
             timeout.CancelAfter(TimeSpan.FromSeconds(20.0));
             try
             {
                 await Task.WhenAll(usedProviders
-                                    .Select(p => p.Fetcher.UpdateIfNecessary(timeout.Token).ContinueWith(t =>
+                    .Select(p => p.Fetcher.UpdateIfNecessary(timeout.Token))
-                                    {
-                                        if (t.Result.Exception != null && t.Result.Exception is not NotSupportedException)
-                                        {
-                                            Logs.PayServer.LogWarning($"Error while contacting exchange {p.ExchangeName}: {t.Result.Exception.Message}");
-                                        }
-                                    }, TaskScheduler.Default))
                     .ToArray()).WithCancellation(timeout.Token);
             }
-                catch (OperationCanceledException) when (timeout.IsCancellationRequested)
+            catch (OperationCanceledException) when (timeout.IsCancellationRequested && !cancellationToken.IsCancellationRequested)
             {
             }
-                if (_LastCacheDate is DateTimeOffset lastCache)
+            if (_lastCacheDate is DateTimeOffset lastCache)
             {
                 if (DateTimeOffset.UtcNow - lastCache > TimeSpan.FromMinutes(8.0))
                 {
@@ -97,51 +74,47 @@ namespace BTCPayServer.HostedServices
                 await SaveRateCache();
             }
         }
-            await Task.Delay(TimeSpan.FromSeconds(30), CancellationToken);
-        }
 
-        public override async Task StartAsync(CancellationToken cancellationToken)
+        public async Task StartAsync(CancellationToken cancellationToken)
         {
             await TryLoadRateCache();
-            await base.StartAsync(cancellationToken);
         }
-        public override async Task StopAsync(CancellationToken cancellationToken)
+        public async Task StopAsync(CancellationToken cancellationToken)
         {
             await SaveRateCache();
-            await base.StopAsync(cancellationToken);
         }
 
         private async Task TryLoadRateCache()
         {
+            ExchangeRatesCache? cache = null;
             try
             {
-                var cache = await _SettingsRepository.GetSettingAsync<ExchangeRatesCache>();
+                cache = JsonConvert.DeserializeObject<ExchangeRatesCache>(await File.ReadAllTextAsync(GetRatesCacheFilePath(), new UTF8Encoding(false)));
-                if (cache != null)
+            }
+            catch
             {
-                    _LastCacheDate = cache.Created;
+            }
+            if (cache is { States: not null })
+            {
+                _lastCacheDate = cache.Created;
                 var stateByExchange = cache.States.ToDictionary(o => o.ExchangeName);
-                    foreach (var provider in _RateProviderFactory.Providers)
+                foreach (var kv in stateByExchange)
                 {
-                        if (stateByExchange.TryGetValue(provider.Key, out var state) &&
+                    if (rateProviderFactory.Providers.TryGetValue(kv.Key, out var prov) &&
-                            provider.Value is BackgroundFetcherRateProvider fetcher)
+                        prov is BackgroundFetcherRateProvider fetcher)
                     {
-                            fetcher.LoadState(state);
+                        fetcher.LoadState(kv.Value);
                     }
                 }
             }
         }
-            catch (Exception ex)
-            {
-                Logs.PayServer.LogWarning(ex, "Warning: Error while trying to load rates from cache");
-            }
-        }
 
-        DateTimeOffset? _LastCacheDate;
+        DateTimeOffset? _lastCacheDate;
         private async Task SaveRateCache()
         {
             var cache = new ExchangeRatesCache();
             cache.Created = DateTimeOffset.UtcNow;
-            _LastCacheDate = cache.Created;
+            _lastCacheDate = cache.Created;
 
             var usedProviders = GetStillUsedProviders().ToArray();
             cache.States = new List<BackgroundFetcherState>(usedProviders.Length);
@@ -151,7 +124,10 @@ namespace BTCPayServer.HostedServices
                 state.ExchangeName = provider.ExchangeName;
                 cache.States.Add(state);
             }
-            await _SettingsRepository.UpdateSetting(cache);
+
+            await File.WriteAllTextAsync(GetRatesCacheFilePath(), JsonConvert.SerializeObject(cache), new UTF8Encoding(false));
         }
+
+        private string GetRatesCacheFilePath() => Path.Combine(dataDirectories.Value.DataDir, "rates-cache.json");
     }
 }

BTCPayServer/Hosting/BTCPayServerServices.cs

@@ -427,7 +427,9 @@ o.GetRequiredService<IEnumerable<IPaymentLinkExtension>>().ToDictionary(o => o.P
             services.AddSingleton<IHostedService, InvoiceEventSaverService>();
             services.AddSingleton<IHostedService, BitpayIPNSender>();
             services.AddSingleton<IHostedService, InvoiceWatcher>();
-            services.AddSingleton<IHostedService, RatesHostedService>();
+            services.AddSingleton<RatesHostedService>();
+            services.AddSingleton<IHostedService>(s => s.GetRequiredService<RatesHostedService>());
+            services.AddScheduledTask<RatesHostedService>(TimeSpan.FromSeconds(30));
             services.AddSingleton<IHostedService, BackgroundJobSchedulerHostedService>();
             services.AddSingleton<IHostedService, AppHubStreamer>();
             services.AddSingleton<IHostedService, AppInventoryUpdaterHostedService>();