diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs index c46a28f18..8eca4b353 100644 --- a/BTCPayServer.Tests/UnitTest1.cs +++ b/BTCPayServer.Tests/UnitTest1.cs @@ -1687,6 +1687,8 @@ namespace BTCPayServer.Tests && e.BidAsk.Bid > 1.0m // 1BTC will always be more than 1USD ); } + // Kraken emit one request only after first GetRates + factory.DirectProviders["kraken"].GetRatesAsync().GetAwaiter().GetResult(); } [Fact] @@ -1711,7 +1713,7 @@ namespace BTCPayServer.Tests private static BTCPayRateProviderFactory CreateBTCPayRateFactory(BTCPayNetworkProvider provider) { - return new BTCPayRateProviderFactory(new MemoryCacheOptions() { ExpirationScanFrequency = TimeSpan.FromSeconds(1.0) }, provider, new CoinAverageSettings()); + return new BTCPayRateProviderFactory(new MemoryCacheOptions() { ExpirationScanFrequency = TimeSpan.FromSeconds(1.0) }, null, provider, new CoinAverageSettings()); } [Fact] diff --git a/BTCPayServer.Tests/docker-compose.yml b/BTCPayServer.Tests/docker-compose.yml index 2e093e43e..7a5654eaf 100644 --- a/BTCPayServer.Tests/docker-compose.yml +++ b/BTCPayServer.Tests/docker-compose.yml @@ -63,7 +63,7 @@ services: nbxplorer: - image: nicolasdorier/nbxplorer:1.0.2.14 + image: nicolasdorier/nbxplorer:1.0.2.31 ports: - "32838:32838" expose: diff --git a/BTCPayServer/BTCPayServer.csproj b/BTCPayServer/BTCPayServer.csproj index c5e30e234..26ebe0aed 100644 --- a/BTCPayServer/BTCPayServer.csproj +++ b/BTCPayServer/BTCPayServer.csproj @@ -2,9 +2,12 @@ Exe netcoreapp2.1 - 1.0.2.93 + 1.0.2.94 NU1701,CA1816,CA1308,CA1810,CA2208 + + 7.3 + @@ -31,22 +34,22 @@ - + - + - + - + - + diff --git a/BTCPayServer/Controllers/InvoiceController.cs b/BTCPayServer/Controllers/InvoiceController.cs index 1b8932914..0add952df 100644 --- a/BTCPayServer/Controllers/InvoiceController.cs +++ b/BTCPayServer/Controllers/InvoiceController.cs @@ -210,6 +210,7 @@ namespace BTCPayServer.Controllers try { var storeBlob = store.GetStoreBlob(); + var preparePayment = handler.PreparePayment(supportedPaymentMethod, store, network); var rate = await fetchingByCurrencyPair[new CurrencyPair(network.CryptoCode, entity.ProductInformation.Currency)]; if (rate.BidAsk == null) { @@ -220,7 +221,7 @@ namespace BTCPayServer.Controllers paymentMethod.Network = network; paymentMethod.SetId(supportedPaymentMethod.PaymentId); paymentMethod.Rate = rate.BidAsk.Bid; - var paymentDetails = await handler.CreatePaymentMethodDetails(supportedPaymentMethod, paymentMethod, store, network); + var paymentDetails = await handler.CreatePaymentMethodDetails(supportedPaymentMethod, paymentMethod, store, network, preparePayment); if (storeBlob.NetworkFeeDisabled) paymentDetails.SetNoTxFee(); paymentMethod.SetPaymentMethodDetails(paymentDetails); diff --git a/BTCPayServer/Hosting/BTCPayServerServices.cs b/BTCPayServer/Hosting/BTCPayServerServices.cs index 566cef736..20f556514 100644 --- a/BTCPayServer/Hosting/BTCPayServerServices.cs +++ b/BTCPayServer/Hosting/BTCPayServerServices.cs @@ -53,6 +53,7 @@ namespace BTCPayServer.Hosting var factory = provider.GetRequiredService(); factory.ConfigureBuilder(o); }); + services.AddHttpClient(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(o => o.GetRequiredService>().Value); diff --git a/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs b/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs index 4ddb426aa..f38bd84f6 100644 --- a/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs +++ b/BTCPayServer/Payments/Bitcoin/BitcoinLikePaymentHandler.cs @@ -27,16 +27,30 @@ namespace BTCPayServer.Payments.Bitcoin _WalletProvider = walletProvider; } - public override async Task CreatePaymentMethodDetails(DerivationStrategy supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network) + class Prepare + { + public Task GetFeeRate; + public Task ReserveAddress; + } + + public override object PreparePayment(DerivationStrategy supportedPaymentMethod, StoreData store, BTCPayNetwork network) + { + return new Prepare() + { + GetFeeRate = _FeeRateProviderFactory.CreateFeeProvider(network).GetFeeRateAsync(), + ReserveAddress = _WalletProvider.GetWallet(network).ReserveAddressAsync(supportedPaymentMethod.DerivationStrategyBase) + }; + } + + public override async Task CreatePaymentMethodDetails(DerivationStrategy supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network, object preparePaymentObject) { if (!_ExplorerProvider.IsAvailable(network)) throw new PaymentMethodUnavailableException($"Full node not available"); - var getFeeRate = _FeeRateProviderFactory.CreateFeeProvider(network).GetFeeRateAsync(); - var getAddress = _WalletProvider.GetWallet(network).ReserveAddressAsync(supportedPaymentMethod.DerivationStrategyBase); + var prepare = (Prepare)preparePaymentObject; Payments.Bitcoin.BitcoinLikeOnChainPaymentMethod onchainMethod = new Payments.Bitcoin.BitcoinLikeOnChainPaymentMethod(); - onchainMethod.FeeRate = await getFeeRate; + onchainMethod.FeeRate = await prepare.GetFeeRate; onchainMethod.TxFee = onchainMethod.FeeRate.GetFee(100); // assume price for 100 bytes - onchainMethod.DepositAddress = (await getAddress).ToString(); + onchainMethod.DepositAddress = (await prepare.ReserveAddress).ToString(); return onchainMethod; } } diff --git a/BTCPayServer/Payments/IPaymentMethodHandler.cs b/BTCPayServer/Payments/IPaymentMethodHandler.cs index 9ec09fa45..11687caef 100644 --- a/BTCPayServer/Payments/IPaymentMethodHandler.cs +++ b/BTCPayServer/Payments/IPaymentMethodHandler.cs @@ -20,23 +20,46 @@ namespace BTCPayServer.Payments /// /// /// - Task CreatePaymentMethodDetails(ISupportedPaymentMethod supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network); + Task CreatePaymentMethodDetails(ISupportedPaymentMethod supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network, object preparePaymentObject); + + /// + /// This method called before the rate have been fetched + /// + /// + /// + /// + /// + object PreparePayment(ISupportedPaymentMethod supportedPaymentMethod, StoreData store, BTCPayNetwork network); } public interface IPaymentMethodHandler : IPaymentMethodHandler where T : ISupportedPaymentMethod { - Task CreatePaymentMethodDetails(T supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network); + Task CreatePaymentMethodDetails(T supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network, object preparePaymentObject); } public abstract class PaymentMethodHandlerBase : IPaymentMethodHandler where T : ISupportedPaymentMethod { - public abstract Task CreatePaymentMethodDetails(T supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network); + + public abstract Task CreatePaymentMethodDetails(T supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network, object preparePaymentObject); + public virtual object PreparePayment(T supportedPaymentMethod, StoreData store, BTCPayNetwork network) + { + return null; + } - Task IPaymentMethodHandler.CreatePaymentMethodDetails(ISupportedPaymentMethod supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network) + object IPaymentMethodHandler.PreparePayment(ISupportedPaymentMethod supportedPaymentMethod, StoreData store, BTCPayNetwork network) { if (supportedPaymentMethod is T method) { - return CreatePaymentMethodDetails(method, paymentMethod, store, network); + return PreparePayment(method, store, network); + } + throw new NotSupportedException("Invalid supportedPaymentMethod"); + } + + Task IPaymentMethodHandler.CreatePaymentMethodDetails(ISupportedPaymentMethod supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network, object preparePaymentObject) + { + if (supportedPaymentMethod is T method) + { + return CreatePaymentMethodDetails(method, paymentMethod, store, network, preparePaymentObject); } throw new NotSupportedException("Invalid supportedPaymentMethod"); } diff --git a/BTCPayServer/Payments/Lightning/LightningLikePaymentHandler.cs b/BTCPayServer/Payments/Lightning/LightningLikePaymentHandler.cs index 4b6c263fd..21a37226a 100644 --- a/BTCPayServer/Payments/Lightning/LightningLikePaymentHandler.cs +++ b/BTCPayServer/Payments/Lightning/LightningLikePaymentHandler.cs @@ -26,7 +26,7 @@ namespace BTCPayServer.Payments.Lightning _LightningClientFactory = lightningClientFactory; _Dashboard = dashboard; } - public override async Task CreatePaymentMethodDetails(LightningSupportedPaymentMethod supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network) + public override async Task CreatePaymentMethodDetails(LightningSupportedPaymentMethod supportedPaymentMethod, PaymentMethod paymentMethod, StoreData store, BTCPayNetwork network, object preparePaymentObject) { var storeBlob = store.GetStoreBlob(); var test = Test(supportedPaymentMethod, network); diff --git a/BTCPayServer/Rating/ExchangeRates.cs b/BTCPayServer/Rating/ExchangeRates.cs index a69cfe4dd..6b3642d59 100644 --- a/BTCPayServer/Rating/ExchangeRates.cs +++ b/BTCPayServer/Rating/ExchangeRates.cs @@ -221,6 +221,16 @@ namespace BTCPayServer.Rating } public class ExchangeRate { + public ExchangeRate() + { + + } + public ExchangeRate(string exchange, CurrencyPair currencyPair, BidAsk bidAsk) + { + this.Exchange = exchange; + this.CurrencyPair = currencyPair; + this.BidAsk = bidAsk; + } public string Exchange { get; set; } public CurrencyPair CurrencyPair { get; set; } public BidAsk BidAsk { get; set; } diff --git a/BTCPayServer/Services/Rates/BTCPayRateProviderFactory.cs b/BTCPayServer/Services/Rates/BTCPayRateProviderFactory.cs index fd378d1c4..d409000f8 100644 --- a/BTCPayServer/Services/Rates/BTCPayRateProviderFactory.cs +++ b/BTCPayServer/Services/Rates/BTCPayRateProviderFactory.cs @@ -36,6 +36,8 @@ namespace BTCPayServer.Services.Rates } IMemoryCache _Cache; private IOptions _CacheOptions; + private readonly IHttpClientFactory _httpClientFactory; + public IMemoryCache Cache { get @@ -45,6 +47,7 @@ namespace BTCPayServer.Services.Rates } CoinAverageSettings _CoinAverageSettings; public BTCPayRateProviderFactory(IOptions cacheOptions, + IHttpClientFactory httpClientFactory, BTCPayNetworkProvider btcpayNetworkProvider, CoinAverageSettings coinAverageSettings) { @@ -53,6 +56,7 @@ namespace BTCPayServer.Services.Rates _CoinAverageSettings = coinAverageSettings; _Cache = new MemoryCache(cacheOptions); _CacheOptions = cacheOptions; + _httpClientFactory = httpClientFactory; // We use 15 min because of limits with free version of bitcoinaverage CacheSpan = TimeSpan.FromMinutes(15.0); this.btcpayNetworkProvider = btcpayNetworkProvider; @@ -66,17 +70,17 @@ namespace BTCPayServer.Services.Rates // We need to be careful to only add exchanges which OnGetTickers implementation make only 1 request DirectProviders.Add("binance", new ExchangeSharpRateProvider("binance", new ExchangeBinanceAPI(), true)); DirectProviders.Add("bittrex", new ExchangeSharpRateProvider("bittrex", new ExchangeBittrexAPI(), true)); - DirectProviders.Add("poloniex", new ExchangeSharpRateProvider("poloniex", new ExchangePoloniexAPI(), false)); + DirectProviders.Add("poloniex", new ExchangeSharpRateProvider("poloniex", new ExchangePoloniexAPI(), true)); DirectProviders.Add("hitbtc", new ExchangeSharpRateProvider("hitbtc", new ExchangeHitbtcAPI(), false)); DirectProviders.Add("cryptopia", new ExchangeSharpRateProvider("cryptopia", new ExchangeCryptopiaAPI(), false)); // Handmade providers DirectProviders.Add("bitpay", new BitpayRateProvider(new NBitpayClient.Bitpay(new NBitcoin.Key(), new Uri("https://bitpay.com/")))); DirectProviders.Add(QuadrigacxRateProvider.QuadrigacxName, new QuadrigacxRateProvider()); - DirectProviders.Add(CoinAverageRateProvider.CoinAverageName, new CoinAverageRateProvider() { Exchange = CoinAverageRateProvider.CoinAverageName, Authenticator = _CoinAverageSettings }); + DirectProviders.Add(CoinAverageRateProvider.CoinAverageName, new CoinAverageRateProvider() { Exchange = CoinAverageRateProvider.CoinAverageName, HttpClient = _httpClientFactory?.CreateClient(), Authenticator = _CoinAverageSettings }); // Those exchanges make multiple requests when calling GetTickers so we remove them - //DirectProviders.Add("kraken", new ExchangeSharpRateProvider("kraken", new ExchangeKrakenAPI(), true)); + DirectProviders.Add("kraken", new KrakenExchangeRateProvider() { HttpClient = _httpClientFactory?.CreateClient() }); //DirectProviders.Add("gdax", new ExchangeSharpRateProvider("gdax", new ExchangeGdaxAPI())); //DirectProviders.Add("gemini", new ExchangeSharpRateProvider("gemini", new ExchangeGeminiAPI())); //DirectProviders.Add("bitfinex", new ExchangeSharpRateProvider("bitfinex", new ExchangeBitfinexAPI())); @@ -194,6 +198,7 @@ namespace BTCPayServer.Services.Rates providers.Add(new CoinAverageRateProvider() { Exchange = exchangeName, + HttpClient = _httpClientFactory?.CreateClient(), Authenticator = _CoinAverageSettings }); } diff --git a/BTCPayServer/Services/Rates/CoinAverageRateProvider.cs b/BTCPayServer/Services/Rates/CoinAverageRateProvider.cs index 1b6e83b01..b0fc7b441 100644 --- a/BTCPayServer/Services/Rates/CoinAverageRateProvider.cs +++ b/BTCPayServer/Services/Rates/CoinAverageRateProvider.cs @@ -56,6 +56,19 @@ namespace BTCPayServer.Services.Rates { } + + public HttpClient HttpClient + { + get + { + return _LocalClient ?? _Client; + } + set + { + _LocalClient = null; + } + } + HttpClient _LocalClient; static HttpClient _Client = new HttpClient(); public string Exchange { get; set; } = CoinAverageName; @@ -107,7 +120,7 @@ namespace BTCPayServer.Services.Rates { await auth.AddHeader(request); } - var resp = await _Client.SendAsync(request); + var resp = await HttpClient.SendAsync(request); using (resp) { @@ -150,7 +163,7 @@ namespace BTCPayServer.Services.Rates { await auth.AddHeader(request); } - var resp = await _Client.SendAsync(request); + var resp = await HttpClient.SendAsync(request); resp.EnsureSuccessStatusCode(); } @@ -162,7 +175,7 @@ namespace BTCPayServer.Services.Rates { await auth.AddHeader(request); } - var resp = await _Client.SendAsync(request); + var resp = await HttpClient.SendAsync(request); resp.EnsureSuccessStatusCode(); var jobj = JObject.Parse(await resp.Content.ReadAsStringAsync()); var response = new GetRateLimitsResponse(); @@ -193,7 +206,7 @@ namespace BTCPayServer.Services.Rates { await auth.AddHeader(request); } - var resp = await _Client.SendAsync(request); + var resp = await HttpClient.SendAsync(request); resp.EnsureSuccessStatusCode(); var jobj = JObject.Parse(await resp.Content.ReadAsStringAsync()); var response = new GetExchangeTickersResponse(); diff --git a/BTCPayServer/Services/Rates/ExchangeSharpRateProvider.cs b/BTCPayServer/Services/Rates/ExchangeSharpRateProvider.cs index 6efdad8d3..230749db2 100644 --- a/BTCPayServer/Services/Rates/ExchangeSharpRateProvider.cs +++ b/BTCPayServer/Services/Rates/ExchangeSharpRateProvider.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -43,17 +44,17 @@ namespace BTCPayServer.Services.Rates } // ExchangeSymbolToGlobalSymbol throws exception which would kill perf - HashSet notFoundSymbols = new HashSet(); + ConcurrentDictionary notFoundSymbols = new ConcurrentDictionary(); private ExchangeRate CreateExchangeRate(KeyValuePair ticker) { - if (notFoundSymbols.Contains(ticker.Key)) + if (notFoundSymbols.ContainsKey(ticker.Key)) return null; try { var tickerName = _ExchangeAPI.ExchangeSymbolToGlobalSymbol(ticker.Key); if (!CurrencyPair.TryParse(tickerName, out var pair)) { - notFoundSymbols.Add(ticker.Key); + notFoundSymbols.TryAdd(ticker.Key, ticker.Key); return null; } if(ReverseCurrencyPair) @@ -66,7 +67,7 @@ namespace BTCPayServer.Services.Rates } catch (ArgumentException) { - notFoundSymbols.Add(ticker.Key); + notFoundSymbols.TryAdd(ticker.Key, ticker.Key); return null; } } diff --git a/BTCPayServer/Services/Rates/KrakenExchangeRateProvider.cs b/BTCPayServer/Services/Rates/KrakenExchangeRateProvider.cs new file mode 100644 index 000000000..af95d530a --- /dev/null +++ b/BTCPayServer/Services/Rates/KrakenExchangeRateProvider.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using BTCPayServer.Rating; +using ExchangeSharp; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace BTCPayServer.Services.Rates +{ + // Make sure that only one request is sent to kraken in general + public class KrakenExchangeRateProvider : IRateProvider + { + public KrakenExchangeRateProvider() + { + _Helper = new ExchangeKrakenAPI(); + } + ExchangeKrakenAPI _Helper; + public HttpClient HttpClient + { + get + { + return _LocalClient ?? _Client; + } + set + { + _LocalClient = null; + } + } + HttpClient _LocalClient; + static HttpClient _Client = new HttpClient(); + + // ExchangeSymbolToGlobalSymbol throws exception which would kill perf + ConcurrentDictionary notFoundSymbols = new ConcurrentDictionary(); + string[] _Symbols = Array.Empty(); + DateTimeOffset? _LastSymbolUpdate = null; + + public async Task GetRatesAsync() + { + var result = new ExchangeRates(); + var symbols = await GetSymbolsAsync(); + var normalizedPairsList = symbols.Where(s => !notFoundSymbols.ContainsKey(s)).Select(s => _Helper.NormalizeSymbol(s)).ToList(); + var csvPairsList = string.Join(",", normalizedPairsList); + JToken apiTickers = await MakeJsonRequestAsync("/0/public/Ticker", null, new Dictionary { { "pair", csvPairsList } }); + var tickers = new List>(); + foreach (string symbol in symbols) + { + var ticker = ConvertToExchangeTicker(symbol, apiTickers[symbol]); + if (ticker != null) + { + try + { + var global = _Helper.ExchangeSymbolToGlobalSymbol(symbol); + if (CurrencyPair.TryParse(global, out var pair)) + result.Add(new ExchangeRate("kraken", pair.Inverse(), new BidAsk(ticker.Bid, ticker.Ask))); + else + notFoundSymbols.TryAdd(symbol, symbol); + } + catch (ArgumentException) + { + notFoundSymbols.TryAdd(symbol, symbol); + } + } + } + return result; + } + + private static ExchangeTicker ConvertToExchangeTicker(string symbol, JToken ticker) + { + if (ticker == null) + return null; + decimal last = ticker["c"][0].ConvertInvariant(); + return new ExchangeTicker + { + Ask = ticker["a"][0].ConvertInvariant(), + Bid = ticker["b"][0].ConvertInvariant(), + Last = last, + Volume = new ExchangeVolume + { + BaseVolume = ticker["v"][1].ConvertInvariant(), + BaseSymbol = symbol, + ConvertedVolume = ticker["v"][1].ConvertInvariant() * last, + ConvertedSymbol = symbol, + Timestamp = DateTime.UtcNow + } + }; + } + + private async Task GetSymbolsAsync() + { + if (_LastSymbolUpdate != null && DateTimeOffset.UtcNow - _LastSymbolUpdate.Value < TimeSpan.FromDays(0.5)) + { + return _Symbols; + } + else + { + JToken json = await MakeJsonRequestAsync("/0/public/AssetPairs"); + var symbols = (from prop in json.Children() where !prop.Name.Contains(".d", StringComparison.OrdinalIgnoreCase) select prop.Name).ToArray(); + _Symbols = symbols; + _LastSymbolUpdate = DateTimeOffset.UtcNow; + return symbols; + } + } + + private async Task MakeJsonRequestAsync(string url, string baseUrl = null, Dictionary payload = null, string requestMethod = null) + { + StringBuilder sb = new StringBuilder(); + sb.Append("https://api.kraken.com"); + ; + sb.Append(url); + if (payload != null) + { + sb.Append("?"); + sb.Append(String.Join('&', payload.Select(kv => $"{kv.Key}={kv.Value}").OfType().ToArray())); + } + var request = new HttpRequestMessage(HttpMethod.Get, sb.ToString()); + var response = await HttpClient.SendAsync(request); + string stringResult = await response.Content.ReadAsStringAsync(); + var result = JsonConvert.DeserializeObject(stringResult); + if (result is JToken json) + { + if (!(json is JArray) && json["error"] is JArray error && error.Count != 0) + { + throw new APIException(error[0].ToStringInvariant()); + } + result = (T)(object)(json["result"] ?? json); + } + return result; + } + } +}