aljaz/btcpayserver
1
0
Fork 0
mirror of https://github.com/aljazceru/btcpayserver.git synced 2026-02-23 15:14:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Do not preemptively fetch rates of all exchanges

Browse Source
This commit is contained in:
nicolas.dorier
2019-12-26 14:22:36 +09:00
parent 4d7480db15
commit 731341b749
5 changed files with 235 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
BTCPayServer.Rating/Providers/BackgroundFetcherRateProvider.cs
View File

@@ -8,16 +8,61 @@ using BTCPayServer.Rating;
using System.Threading;
using Microsoft.Extensions.Logging.Abstractions;
using BTCPayServer.Logging;
using Newtonsoft.Json;
using System.Reflection;
using System.Globalization;
namespace BTCPayServer.Services.Rates
{
public class BackgroundFetcherState
{
public string ExchangeName { get; set; }
[JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))]
public DateTimeOffset? LastRequested { get; set; }
[JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))]
public DateTimeOffset? NextUpdate { get; set; }
[JsonProperty(ItemConverterType = typeof(BackgroundFetcherRateJsonConverter))]
public List<BackgroundFetcherRate> Rates { get; set; }
}
public class BackgroundFetcherRate
{
public CurrencyPair Pair { get; set; }
public BidAsk BidAsk { get; set; }
}
//This make the json more compact
class BackgroundFetcherRateJsonConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
{
return typeof(BackgroundFetcherRate).GetTypeInfo().IsAssignableFrom(objectType.GetTypeInfo());
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
var value = (string)reader.Value;
var parts = value.Split('|');
return new BackgroundFetcherRate()
{
Pair = CurrencyPair.Parse(parts[0]),
BidAsk = new BidAsk(decimal.Parse(parts[1], CultureInfo.InvariantCulture), decimal.Parse(parts[2], CultureInfo.InvariantCulture))
};
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
var rate = (BackgroundFetcherRate)value;
writer.WriteValue($"{rate.Pair}|{rate.BidAsk.Bid.ToString(CultureInfo.InvariantCulture)}|{rate.BidAsk.Ask.ToString(CultureInfo.InvariantCulture)}");
}
}
/// <summary>
/// This class is a decorator which handle caching and pre-emptive query to the underlying exchange
/// </summary>
public class BackgroundFetcherRateProvider : IRateProvider
{
public class LatestFetch
{
public ExchangeRates Latest;
public DateTimeOffset NextRefresh;
public TimeSpan Backoff = TimeSpan.FromSeconds(5.0);
public TimeSpan Backoff = TimeSpan.FromSeconds(5.0);
public DateTimeOffset Expiration;
public Exception Exception;
public string ExchangeName;
@@ -39,12 +84,56 @@ namespace BTCPayServer.Services.Rates
}
IRateProvider _Inner;
public IRateProvider Inner => _Inner;
public BackgroundFetcherRateProvider(IRateProvider inner)
public BackgroundFetcherRateProvider(string exchangeName, IRateProvider inner)
{
if (inner == null)
throw new ArgumentNullException(nameof(inner));
if (exchangeName == null)
throw new ArgumentNullException(nameof(exchangeName));
_Inner = inner;
ExchangeName = exchangeName;
}
public BackgroundFetcherState GetState()
{
var state = new BackgroundFetcherState()
{
ExchangeName = ExchangeName,
LastRequested = LastRequested
};
if (_Latest is LatestFetch fetch)
{
state.NextUpdate = fetch.NextRefresh;
state.Rates = fetch.Latest
.Where(e => e.Exchange == ExchangeName)
.Select(r => new BackgroundFetcherRate()
{
Pair = r.CurrencyPair,
BidAsk = r.BidAsk
}).ToList();
}
return state;
}
public void LoadState(BackgroundFetcherState state)
{
if (ExchangeName != state.ExchangeName)
throw new InvalidOperationException("The state does not belong to this fetcher");
if (state.LastRequested is DateTimeOffset lastRequested)
this.LastRequested = state.LastRequested;
if (state.NextUpdate is DateTimeOffset nextUpdate && state.Rates is List<BackgroundFetcherRate> rates)
{
var fetch = new LatestFetch()
{
ExchangeName = state.ExchangeName,
Latest = new ExchangeRates(rates.Select(r => new ExchangeRate(state.ExchangeName, r.Pair, r.BidAsk))),
NextRefresh = nextUpdate,
Expiration = nextUpdate - RefreshRate + ValidatyTime
};
_Latest = fetch;
}
}
TimeSpan _RefreshRate = TimeSpan.FromSeconds(30);
@@ -115,24 +204,36 @@ namespace BTCPayServer.Services.Rates
var latest = _Latest;
if (!DoNotAutoFetchIfExpired && latest != null && latest.Expiration <= DateTimeOffset.UtcNow + TimeSpan.FromSeconds(1.0))
{
Logs.PayServer.LogWarning($"GetRatesAsync was called on {GetExchangeName()} when the rate is outdated. It should never happen, let BTCPayServer developers know about this.");
latest = null;
}
LastRequested = DateTimeOffset.UtcNow;
return (latest ?? (await Fetch(cancellationToken))).GetResult();
}
private string GetExchangeName()
/// <summary>
/// The last time this rate provider has been used
/// </summary>
public DateTimeOffset? LastRequested { get; set; }
public string ExchangeName { get; }
public DateTimeOffset? Expiration
{
if (_Inner is IHasExchangeName exchangeName)
return exchangeName.ExchangeName ?? "???";
return "???";
get
{
if (_Latest is LatestFetch f)
{
return f.Expiration;
}
return null;
}
}
private async Task<LatestFetch> Fetch(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var previous = _Latest;
var fetch = new LatestFetch();
fetch.ExchangeName = GetExchangeName();
fetch.ExchangeName = ExchangeName;
try
{
var rates = await _Inner.GetRatesAsync(cancellationToken);

2
BTCPayServer.Rating/Services/RateProviderFactory.cs
View File

@@ -128,7 +128,7 @@ namespace BTCPayServer.Services.Rates
{
if (provider.Key == "cryptopia") // Shitty exchange, rate often unavailable, it spams the logs
continue;
var prov = new BackgroundFetcherRateProvider(Providers[provider.Key]);
var prov = new BackgroundFetcherRateProvider(provider.Key, Providers[provider.Key]);
if(provider.Key == CoinAverageRateProvider.CoinAverageName)
{
prov.RefreshRate = CacheSpan;

41
BTCPayServer.Tests/UnitTest1.cs
View File

@@ -1844,7 +1844,7 @@ namespace BTCPayServer.Tests
}
}
[Fact(Timeout = 60 * 2 * 1000)]
[Fact]
[Trait("Integration", "Integration")]
public async Task CanSetPaymentMethodLimits()
{
@@ -2694,6 +2694,43 @@ noninventoryitem:
factory.Providers["kraken"].GetRatesAsync(default).GetAwaiter().GetResult();
}
[Fact(Timeout = TestTimeout)]
[Trait("Integration", "Integration")]
public async Task CanExportBackgroundFetcherState()
{
var factory = CreateBTCPayRateFactory();
var provider = (BackgroundFetcherRateProvider)factory.Providers["kraken"];
await provider.GetRatesAsync(default);
var state = provider.GetState();
Assert.Single(state.Rates, r => r.Pair == new CurrencyPair("BTC", "EUR"));
var provider2 = new BackgroundFetcherRateProvider("kraken", provider.Inner);
using (var cts = new CancellationTokenSource())
{
cts.Cancel();
// Should throw
await Assert.ThrowsAsync<OperationCanceledException>(async () => await provider2.GetRatesAsync(cts.Token));
}
provider2.LoadState(state);
Assert.Equal(provider.LastRequested, provider2.LastRequested);
using (var cts = new CancellationTokenSource())
{
cts.Cancel();
// Should not throw, as things should be cached
await provider2.GetRatesAsync(cts.Token);
}
Assert.Equal(provider.ExchangeName, provider2.ExchangeName);
Assert.Equal(provider.NextUpdate, provider2.NextUpdate);
Assert.NotEqual(provider.LastRequested, provider2.LastRequested);
Assert.NotEqual(provider.Expiration, provider2.Expiration);
var str = JsonConvert.SerializeObject(state);
var state2 = JsonConvert.DeserializeObject<BackgroundFetcherState>(str);
var str2 = JsonConvert.SerializeObject(state2);
Assert.Equal(str, str2);
}
[Fact(Timeout = TestTimeout)]
[Trait("Integration", "Integration")]
public void CanGetRateCryptoCurrenciesByDefault()
@@ -2859,7 +2896,7 @@ noninventoryitem:
spy.AssertHit();
factory.Providers.Clear();
var fetch = new BackgroundFetcherRateProvider(spy);
var fetch = new BackgroundFetcherRateProvider("spy", spy);
fetch.DoNotAutoFetchIfExpired = true;
factory.Providers.Add("bittrex", fetch);
fetchedRate = fetcher.FetchRate(CurrencyPair.Parse("BTC_USD"), rateRules, default).GetAwaiter().GetResult();

4
BTCPayServer/HostedServices/BaseAsyncService.cs
View File

@@ -19,7 +19,7 @@ namespace BTCPayServer.HostedServices
private CancellationTokenSource _Cts;
protected Task[] _Tasks;
public Task StartAsync(CancellationToken cancellationToken)
public virtual Task StartAsync(CancellationToken cancellationToken)
{
_Cts = new CancellationTokenSource();
_Tasks = InitializeTasks();
@@ -57,7 +57,7 @@ namespace BTCPayServer.HostedServices
}
}
public async Task StopAsync(CancellationToken cancellationToken)
public virtual async Task StopAsync(CancellationToken cancellationToken)
{
if (_Cts != null)
{

88
BTCPayServer/HostedServices/RatesHostedService.cs
View File

@@ -12,11 +12,22 @@ using BTCPayServer.Logging;
using System.Runtime.CompilerServices;
using System.IO;
using System.Text;
using Newtonsoft.Json;
namespace BTCPayServer.HostedServices
{
public class RatesHostedService : BaseAsyncService
{
public class ExchangeRatesCache
{
[JsonConverter(typeof(NBitcoin.JsonConverters.DateTimeToUnixTimeConverter))]
public DateTimeOffset Created { get; set; }
public List<BackgroundFetcherState> States { get; set; }
public override string ToString()
{
return "";
}
}
private SettingsRepository _SettingsRepository;
private CoinAverageSettings _coinAverageSettings;
RateProviderFactory _RateProviderFactory;
@@ -38,17 +49,31 @@ namespace BTCPayServer.HostedServices
CreateLoopTask(RefreshRates)
};
}
bool IsStillUsed(BackgroundFetcherRateProvider fetcher)
{
return fetcher.LastRequested is DateTimeOffset v &&
DateTimeOffset.UtcNow - v < TimeSpan.FromDays(1.0);
}
async Task RefreshRates()
{
var usedProviders = _RateProviderFactory.Providers
.Select(p => p.Value)
.OfType<BackgroundFetcherRateProvider>()
.Where(IsStillUsed)
.ToArray();
if (usedProviders.Length == 0)
{
await Task.Delay(TimeSpan.FromSeconds(30), Cancellation);
return;
}
using (var timeout = CancellationTokenSource.CreateLinkedTokenSource(Cancellation))
{
timeout.CancelAfter(TimeSpan.FromSeconds(20.0));
try
{
await Task.WhenAll(_RateProviderFactory.Providers
.Select(p => (Fetcher: p.Value as BackgroundFetcherRateProvider, ExchangeName: p.Key)).Where(p => p.Fetcher != null)
.Select(p => p.Fetcher.UpdateIfNecessary(timeout.Token).ContinueWith(t =>
await Task.WhenAll(usedProviders
.Select(p => p.UpdateIfNecessary(timeout.Token).ContinueWith(t =>
{
if (t.Result.Exception != null)
{
@@ -60,10 +85,65 @@ namespace BTCPayServer.HostedServices
catch (OperationCanceledException) when (timeout.IsCancellationRequested)
{
}
if (_LastCacheDate is DateTimeOffset lastCache)
{
if (DateTimeOffset.UtcNow - lastCache > TimeSpan.FromMinutes(8.0))
{
await SaveRateCache();
}
}
else
{
await SaveRateCache();
}
}
await Task.Delay(TimeSpan.FromSeconds(30), Cancellation);
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
await TryLoadRateCache();
await base.StartAsync(cancellationToken);
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
await SaveRateCache();
await base.StopAsync(cancellationToken);
}
private async Task TryLoadRateCache()
{
var cache = await _SettingsRepository.GetSettingAsync<ExchangeRatesCache>();
if (cache != null)
{
_LastCacheDate = cache.Created;
var stateByExchange = cache.States.ToDictionary(o => o.ExchangeName);
foreach (var obj in _RateProviderFactory.Providers
.Select(p => p.Value)
.OfType<BackgroundFetcherRateProvider>()
.Select(v => (Fetcher: v, State: stateByExchange.TryGet(v.ExchangeName)))
.Where(v => v.State != null))
{
obj.Fetcher.LoadState(obj.State);
}
}
}
DateTimeOffset? _LastCacheDate;
private async Task SaveRateCache()
{
var cache = new ExchangeRatesCache();
cache.Created = DateTimeOffset.UtcNow;
_LastCacheDate = cache.Created;
cache.States = _RateProviderFactory.Providers
.Select(p => p.Value)
.OfType<BackgroundFetcherRateProvider>()
.Where(IsStillUsed)
.Select(p => p.GetState())
.ToList();
await _SettingsRepository.UpdateSetting(cache);
}
async Task RefreshCoinAverageSupportedExchanges()
{
var exchanges = new CoinAverageExchanges();