From 5ef41294e4655a4a9921c39dfccf7015a2d165e2 Mon Sep 17 00:00:00 2001 From: Kukks Date: Wed, 23 Mar 2022 15:03:39 +0000 Subject: [PATCH] Optimize github plugin fetching --- .../Controllers/UIServerController.Plugins.cs | 9 +- BTCPayServer/Plugins/PluginService.cs | 96 +++++++++++-------- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/BTCPayServer/Controllers/UIServerController.Plugins.cs b/BTCPayServer/Controllers/UIServerController.Plugins.cs index c33b9c647..395aa8929 100644 --- a/BTCPayServer/Controllers/UIServerController.Plugins.cs +++ b/BTCPayServer/Controllers/UIServerController.Plugins.cs @@ -6,7 +6,6 @@ using BTCPayServer.Abstractions.Contracts; using BTCPayServer.Abstractions.Extensions; using BTCPayServer.Abstractions.Models; using BTCPayServer.Configuration; -using BTCPayServer.Models; using BTCPayServer.Plugins; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; @@ -127,8 +126,12 @@ namespace BTCPayServer.Controllers StringComparison.InvariantCultureIgnoreCase)); } - return RedirectToAction("ListPlugins", - new { StatusMessage = "Files uploaded, restart server to load plugins" }); + TempData.SetStatusMessageModel(new StatusMessageModel() + { + Message = "Files uploaded, restart server to load plugins" , + Severity = StatusMessageModel.StatusSeverity.Success + }); + return RedirectToAction("ListPlugins"); } } } diff --git a/BTCPayServer/Plugins/PluginService.cs b/BTCPayServer/Plugins/PluginService.cs index 1fa0d4af0..95c755a93 100644 --- a/BTCPayServer/Plugins/PluginService.cs +++ b/BTCPayServer/Plugins/PluginService.cs @@ -4,79 +4,97 @@ using System.IO; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Text; using System.Threading.Tasks; using BTCPayServer.Abstractions.Contracts; -using BTCPayServer.Abstractions.Extensions; using BTCPayServer.Configuration; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; +using NBitcoin.DataEncoders; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace BTCPayServer.Plugins { public class PluginService { private readonly IOptions _dataDirectories; + private readonly IMemoryCache _memoryCache; + private readonly ISettingsRepository _settingsRepository; private readonly BTCPayServerOptions _btcPayServerOptions; private readonly HttpClient _githubClient; - public PluginService(IEnumerable btcPayServerPlugins, - IHttpClientFactory httpClientFactory, BTCPayServerOptions btcPayServerOptions, IOptions dataDirectories) + public PluginService( + ISettingsRepository settingsRepository, + IEnumerable btcPayServerPlugins, + IHttpClientFactory httpClientFactory, BTCPayServerOptions btcPayServerOptions, + IOptions dataDirectories, IMemoryCache memoryCache) { LoadedPlugins = btcPayServerPlugins; _githubClient = httpClientFactory.CreateClient(); _githubClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("btcpayserver", "1")); + _settingsRepository = settingsRepository; _btcPayServerOptions = btcPayServerOptions; _dataDirectories = dataDirectories; + _memoryCache = memoryCache; + } + + private async Task CallHttpAndCache(string uri) + { + var cacheTime = TimeSpan.FromMinutes(30); + return await _memoryCache.GetOrCreateAsync(nameof(PluginService) + uri, async entry => + { + entry.AbsoluteExpiration = DateTimeOffset.UtcNow + cacheTime; + return await _githubClient.GetStringAsync(uri); + }); } public IEnumerable LoadedPlugins { get; } - public async Task GetRemotePlugins(string path = "") + public async Task GetRemotePlugins() { - var resp = await _githubClient - .GetStringAsync(new Uri($"https://api.github.com/repos/{_btcPayServerOptions.PluginRemote}/contents/{path}")); - var files = JsonConvert.DeserializeObject(resp); - var dirs = files.Where(file => file.Type == "dir"); - var result = dirs.Select(file => GetRemotePlugins(file.Path)); + var resp = await CallHttpAndCache($"https://api.github.com/repos/{_btcPayServerOptions.PluginRemote}/git/trees/master?recursive=1"); - var fileTask = Task.WhenAll(files - .Where(file => file.Type == "file" && file.Name.EndsWith($"{PluginManager.BTCPayPluginSuffix}.json", - StringComparison.InvariantCulture)).Select(async file => + var respObj = JObject.Parse(resp)["tree"] as JArray; + + var detectedPlugins = respObj.Where(token => token["path"].ToString().EndsWith(".btcpay")); + + List> result = new List>(); + foreach (JToken detectedPlugin in detectedPlugins) + { + var pluginName = detectedPlugin["path"].ToString(); + + var metadata = respObj.SingleOrDefault(token => (pluginName + ".json")== token["path"].ToString()); + if (metadata is null) { - return await _githubClient.GetStringAsync(file.DownloadUrl).ContinueWith( - task => - { - var r = JsonConvert.DeserializeObject(task.Result); - r.Path = path; - return r; - }, TaskScheduler.Current); - })); - return (await Task.WhenAll( result.Concat(new[] { fileTask })).ContinueWith(task => task.Result.SelectMany(plugins => plugins))).ToArray(); - } - - + continue; + } + result.Add( CallHttpAndCache(metadata["url"].ToString()) + .ContinueWith( + task => + { + var d = JObject.Parse(task.Result); + var content = Encoders.Base64.DecodeData(d["content"].Value()); + + var r = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(content)); + r.Path = $"https://raw.githubusercontent.com/{_btcPayServerOptions.PluginRemote}/master/{pluginName}"; + return r; + }, TaskScheduler.Current)); + + } + + return await Task.WhenAll(result); + } public async Task DownloadRemotePlugin(string plugin, string path) { var dest = _dataDirectories.Value.PluginDir; - var resp = await _githubClient - .GetStringAsync(new Uri($"https://api.github.com/repos/{_btcPayServerOptions.PluginRemote}/contents/{path}")); - var files = JsonConvert.DeserializeObject(resp); - var ext = files.SingleOrDefault(file => file.Name == $"{plugin}{PluginManager.BTCPayPluginSuffix}"); - if (ext is null) - { - throw new Exception("Plugin not found on remote"); - } - - if (!ext.Name.IsValidFileName()) - { - throw new Exception("Invalid file name"); - } - var filedest = Path.Join(dest, ext.Name); + + var filedest = Path.Join(dest, plugin); Directory.CreateDirectory(Path.GetDirectoryName(filedest)); - using var resp2 = await _githubClient.GetAsync(ext.DownloadUrl); + using var resp2 = await _githubClient.GetAsync(path); using var fs = new FileStream(filedest, FileMode.Create, FileAccess.ReadWrite); await resp2.Content.CopyToAsync(fs); await fs.FlushAsync();