From 731341b749d6007e1003457bac57cf3854fc0ce1 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Thu, 26 Dec 2019 14:22:36 +0900 Subject: [PATCH] Do not preemptively fetch rates of all exchanges --- .../BackgroundFetcherRateProvider.cs | 117 ++++++++++++++++-- .../Services/RateProviderFactory.cs | 2 +- BTCPayServer.Tests/UnitTest1.cs | 41 +++++- .../HostedServices/BaseAsyncService.cs | 4 +- .../HostedServices/RatesHostedService.cs | 88 ++++++++++++- 5 files changed, 235 insertions(+), 17 deletions(-) diff --git a/BTCPayServer.Rating/Providers/BackgroundFetcherRateProvider.cs b/BTCPayServer.Rating/Providers/BackgroundFetcherRateProvider.cs index b923e4839..f37002835 100644 --- a/BTCPayServer.Rating/Providers/BackgroundFetcherRateProvider.cs +++ b/BTCPayServer.Rating/Providers/BackgroundFetcherRateProvider.cs @@ -8,16 +8,61 @@ using BTCPayServer.Rating; using System.Threading; using Microsoft.Extensions.Logging.Abstractions; using BTCPayServer.Logging; +using Newtonsoft.Json; +using System.Reflection; +using System.Globalization; namespace BTCPayServer.Services.Rates { + public class BackgroundFetcherState + { + public string ExchangeName { get; set; } + [JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))] + public DateTimeOffset? LastRequested { get; set; } + [JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))] + public DateTimeOffset? NextUpdate { get; set; } + [JsonProperty(ItemConverterType = typeof(BackgroundFetcherRateJsonConverter))] + public List Rates { get; set; } + } + public class BackgroundFetcherRate + { + public CurrencyPair Pair { get; set; } + public BidAsk BidAsk { get; set; } + } + //This make the json more compact + class BackgroundFetcherRateJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return typeof(BackgroundFetcherRate).GetTypeInfo().IsAssignableFrom(objectType.GetTypeInfo()); + } + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var value = (string)reader.Value; + var parts = value.Split('|'); + return new BackgroundFetcherRate() + { + Pair = CurrencyPair.Parse(parts[0]), + BidAsk = new BidAsk(decimal.Parse(parts[1], CultureInfo.InvariantCulture), decimal.Parse(parts[2], CultureInfo.InvariantCulture)) + }; + } + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var rate = (BackgroundFetcherRate)value; + writer.WriteValue($"{rate.Pair}|{rate.BidAsk.Bid.ToString(CultureInfo.InvariantCulture)}|{rate.BidAsk.Ask.ToString(CultureInfo.InvariantCulture)}"); + } + } + + /// + /// This class is a decorator which handle caching and pre-emptive query to the underlying exchange + /// public class BackgroundFetcherRateProvider : IRateProvider { public class LatestFetch { public ExchangeRates Latest; public DateTimeOffset NextRefresh; - public TimeSpan Backoff = TimeSpan.FromSeconds(5.0); + public TimeSpan Backoff = TimeSpan.FromSeconds(5.0); public DateTimeOffset Expiration; public Exception Exception; public string ExchangeName; @@ -39,12 +84,56 @@ namespace BTCPayServer.Services.Rates } IRateProvider _Inner; + public IRateProvider Inner => _Inner; - public BackgroundFetcherRateProvider(IRateProvider inner) + public BackgroundFetcherRateProvider(string exchangeName, IRateProvider inner) { if (inner == null) throw new ArgumentNullException(nameof(inner)); + if (exchangeName == null) + throw new ArgumentNullException(nameof(exchangeName)); _Inner = inner; + ExchangeName = exchangeName; + } + + public BackgroundFetcherState GetState() + { + var state = new BackgroundFetcherState() + { + ExchangeName = ExchangeName, + LastRequested = LastRequested + }; + if (_Latest is LatestFetch fetch) + { + state.NextUpdate = fetch.NextRefresh; + state.Rates = fetch.Latest + .Where(e => e.Exchange == ExchangeName) + .Select(r => new BackgroundFetcherRate() + { + Pair = r.CurrencyPair, + BidAsk = r.BidAsk + }).ToList(); + } + return state; + } + + public void LoadState(BackgroundFetcherState state) + { + if (ExchangeName != state.ExchangeName) + throw new InvalidOperationException("The state does not belong to this fetcher"); + if (state.LastRequested is DateTimeOffset lastRequested) + this.LastRequested = state.LastRequested; + if (state.NextUpdate is DateTimeOffset nextUpdate && state.Rates is List rates) + { + var fetch = new LatestFetch() + { + ExchangeName = state.ExchangeName, + Latest = new ExchangeRates(rates.Select(r => new ExchangeRate(state.ExchangeName, r.Pair, r.BidAsk))), + NextRefresh = nextUpdate, + Expiration = nextUpdate - RefreshRate + ValidatyTime + }; + _Latest = fetch; + } } TimeSpan _RefreshRate = TimeSpan.FromSeconds(30); @@ -115,24 +204,36 @@ namespace BTCPayServer.Services.Rates var latest = _Latest; if (!DoNotAutoFetchIfExpired && latest != null && latest.Expiration <= DateTimeOffset.UtcNow + TimeSpan.FromSeconds(1.0)) { - Logs.PayServer.LogWarning($"GetRatesAsync was called on {GetExchangeName()} when the rate is outdated. It should never happen, let BTCPayServer developers know about this."); latest = null; } + LastRequested = DateTimeOffset.UtcNow; return (latest ?? (await Fetch(cancellationToken))).GetResult(); } - private string GetExchangeName() + /// + /// The last time this rate provider has been used + /// + public DateTimeOffset? LastRequested { get; set; } + + public string ExchangeName { get; } + public DateTimeOffset? Expiration { - if (_Inner is IHasExchangeName exchangeName) - return exchangeName.ExchangeName ?? "???"; - return "???"; + get + { + if (_Latest is LatestFetch f) + { + return f.Expiration; + } + return null; + } } private async Task Fetch(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); var previous = _Latest; var fetch = new LatestFetch(); - fetch.ExchangeName = GetExchangeName(); + fetch.ExchangeName = ExchangeName; try { var rates = await _Inner.GetRatesAsync(cancellationToken); diff --git a/BTCPayServer.Rating/Services/RateProviderFactory.cs b/BTCPayServer.Rating/Services/RateProviderFactory.cs index 1acda8a19..154b871d9 100644 --- a/BTCPayServer.Rating/Services/RateProviderFactory.cs +++ b/BTCPayServer.Rating/Services/RateProviderFactory.cs @@ -128,7 +128,7 @@ namespace BTCPayServer.Services.Rates { if (provider.Key == "cryptopia") // Shitty exchange, rate often unavailable, it spams the logs continue; - var prov = new BackgroundFetcherRateProvider(Providers[provider.Key]); + var prov = new BackgroundFetcherRateProvider(provider.Key, Providers[provider.Key]); if(provider.Key == CoinAverageRateProvider.CoinAverageName) { prov.RefreshRate = CacheSpan; diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index 411d713d0..d0754a7cd 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -1844,7 +1844,7 @@ namespace BTCPayServer.Tests } } - [Fact(Timeout = 60 * 2 * 1000)] + [Fact] [Trait("Integration", "Integration")] public async Task CanSetPaymentMethodLimits() { @@ -2694,6 +2694,43 @@ noninventoryitem: factory.Providers["kraken"].GetRatesAsync(default).GetAwaiter().GetResult(); } + + [Fact(Timeout = TestTimeout)] + [Trait("Integration", "Integration")] + public async Task CanExportBackgroundFetcherState() + { + var factory = CreateBTCPayRateFactory(); + var provider = (BackgroundFetcherRateProvider)factory.Providers["kraken"]; + await provider.GetRatesAsync(default); + var state = provider.GetState(); + Assert.Single(state.Rates, r => r.Pair == new CurrencyPair("BTC", "EUR")); + var provider2 = new BackgroundFetcherRateProvider("kraken", provider.Inner); + using (var cts = new CancellationTokenSource()) + { + cts.Cancel(); + // Should throw + await Assert.ThrowsAsync(async () => await provider2.GetRatesAsync(cts.Token)); + } + provider2.LoadState(state); + Assert.Equal(provider.LastRequested, provider2.LastRequested); + using (var cts = new CancellationTokenSource()) + { + cts.Cancel(); + // Should not throw, as things should be cached + await provider2.GetRatesAsync(cts.Token); + } + Assert.Equal(provider.ExchangeName, provider2.ExchangeName); + Assert.Equal(provider.NextUpdate, provider2.NextUpdate); + Assert.NotEqual(provider.LastRequested, provider2.LastRequested); + Assert.NotEqual(provider.Expiration, provider2.Expiration); + + var str = JsonConvert.SerializeObject(state); + var state2 = JsonConvert.DeserializeObject(str); + var str2 = JsonConvert.SerializeObject(state2); + Assert.Equal(str, str2); + } + + [Fact(Timeout = TestTimeout)] [Trait("Integration", "Integration")] public void CanGetRateCryptoCurrenciesByDefault() @@ -2859,7 +2896,7 @@ noninventoryitem: spy.AssertHit(); factory.Providers.Clear(); - var fetch = new BackgroundFetcherRateProvider(spy); + var fetch = new BackgroundFetcherRateProvider("spy", spy); fetch.DoNotAutoFetchIfExpired = true; factory.Providers.Add("bittrex", fetch); fetchedRate = fetcher.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules, default).GetAwaiter().GetResult(); diff --git a/BTCPayServer/HostedServices/BaseAsyncService.cs b/BTCPayServer/HostedServices/BaseAsyncService.cs index 7236ae99d..f99298395 100644 --- a/BTCPayServer/HostedServices/BaseAsyncService.cs +++ b/BTCPayServer/HostedServices/BaseAsyncService.cs @@ -19,7 +19,7 @@ namespace BTCPayServer.HostedServices private CancellationTokenSource _Cts; protected Task[] _Tasks; - public Task StartAsync(CancellationToken cancellationToken) + public virtual Task StartAsync(CancellationToken cancellationToken) { _Cts = new CancellationTokenSource(); _Tasks = InitializeTasks(); @@ -57,7 +57,7 @@ namespace BTCPayServer.HostedServices } } - public async Task StopAsync(CancellationToken cancellationToken) + public virtual async Task StopAsync(CancellationToken cancellationToken) { if (_Cts != null) { diff --git a/BTCPayServer/HostedServices/RatesHostedService.cs b/BTCPayServer/HostedServices/RatesHostedService.cs index 830bd9c65..9e7776e67 100644 --- a/BTCPayServer/HostedServices/RatesHostedService.cs +++ b/BTCPayServer/HostedServices/RatesHostedService.cs @@ -12,11 +12,22 @@ using BTCPayServer.Logging; using System.Runtime.CompilerServices; using System.IO; using System.Text; +using Newtonsoft.Json; namespace BTCPayServer.HostedServices { public class RatesHostedService : BaseAsyncService { + public class ExchangeRatesCache + { + [JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))] + public DateTimeOffset Created { get; set; } + public List States { get; set; } + public override string ToString() + { + return ""; + } + } private SettingsRepository _SettingsRepository; private CoinAverageSettings _coinAverageSettings; RateProviderFactory _RateProviderFactory; @@ -38,17 +49,31 @@ namespace BTCPayServer.HostedServices CreateLoopTask(RefreshRates) }; } + + bool IsStillUsed(BackgroundFetcherRateProvider fetcher) + { + return fetcher.LastRequested is DateTimeOffset v && + DateTimeOffset.UtcNow - v < TimeSpan.FromDays(1.0); + } async Task RefreshRates() { - + var usedProviders = _RateProviderFactory.Providers + .Select(p => p.Value) + .OfType() + .Where(IsStillUsed) + .ToArray(); + if (usedProviders.Length == 0) + { + await Task.Delay(TimeSpan.FromSeconds(30), Cancellation); + return; + } using (var timeout = CancellationTokenSource.CreateLinkedTokenSource(Cancellation)) { timeout.CancelAfter(TimeSpan.FromSeconds(20.0)); try { - await Task.WhenAll(_RateProviderFactory.Providers - .Select(p => (Fetcher: p.Value as BackgroundFetcherRateProvider, ExchangeName: p.Key)).Where(p => p.Fetcher != null) - .Select(p => p.Fetcher.UpdateIfNecessary(timeout.Token).ContinueWith(t => + await Task.WhenAll(usedProviders + .Select(p => p.UpdateIfNecessary(timeout.Token).ContinueWith(t => { if (t.Result.Exception != null) { @@ -60,10 +85,65 @@ namespace BTCPayServer.HostedServices catch (OperationCanceledException) when (timeout.IsCancellationRequested) { } + if (_LastCacheDate is DateTimeOffset lastCache) + { + if (DateTimeOffset.UtcNow - lastCache > TimeSpan.FromMinutes(8.0)) + { + await SaveRateCache(); + } + } + else + { + await SaveRateCache(); + } } await Task.Delay(TimeSpan.FromSeconds(30), Cancellation); } + public override async Task StartAsync(CancellationToken cancellationToken) + { + await TryLoadRateCache(); + await base.StartAsync(cancellationToken); + } + public override async Task StopAsync(CancellationToken cancellationToken) + { + await SaveRateCache(); + await base.StopAsync(cancellationToken); + } + + private async Task TryLoadRateCache() + { + var cache = await _SettingsRepository.GetSettingAsync(); + if (cache != null) + { + _LastCacheDate = cache.Created; + var stateByExchange = cache.States.ToDictionary(o => o.ExchangeName); + foreach (var obj in _RateProviderFactory.Providers + .Select(p => p.Value) + .OfType() + .Select(v => (Fetcher: v, State: stateByExchange.TryGet(v.ExchangeName))) + .Where(v => v.State != null)) + { + obj.Fetcher.LoadState(obj.State); + } + } + } + + DateTimeOffset? _LastCacheDate; + private async Task SaveRateCache() + { + var cache = new ExchangeRatesCache(); + cache.Created = DateTimeOffset.UtcNow; + _LastCacheDate = cache.Created; + cache.States = _RateProviderFactory.Providers + .Select(p => p.Value) + .OfType() + .Where(IsStillUsed) + .Select(p => p.GetState()) + .ToList(); + await _SettingsRepository.UpdateSetting(cache); + } + async Task RefreshCoinAverageSupportedExchanges() { var exchanges = new CoinAverageExchanges();