Optimize github plugin fetching

BTCPayServer/Controllers/UIServerController.Plugins.cs

@@ -6,7 +6,6 @@ using BTCPayServer.Abstractions.Contracts;
 using BTCPayServer.Abstractions.Extensions;
 using BTCPayServer.Abstractions.Models;
 using BTCPayServer.Configuration;
-using BTCPayServer.Models;
 using BTCPayServer.Plugins;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
@@ -127,8 +126,12 @@ namespace BTCPayServer.Controllers
                     StringComparison.InvariantCultureIgnoreCase));
             }
 
-            return RedirectToAction("ListPlugins",
+            TempData.SetStatusMessageModel(new StatusMessageModel()
-                new { StatusMessage = "Files uploaded, restart server to load plugins" });
+            {
+                Message = "Files uploaded, restart server to load plugins" ,
+                Severity = StatusMessageModel.StatusSeverity.Success
+            });
+            return RedirectToAction("ListPlugins");
         }
     }
 }

BTCPayServer/Plugins/PluginService.cs

@@ -4,79 +4,97 @@ using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Text;
 using System.Threading.Tasks;
 using BTCPayServer.Abstractions.Contracts;
-using BTCPayServer.Abstractions.Extensions;
 using BTCPayServer.Configuration;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
+using NBitcoin.DataEncoders;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace BTCPayServer.Plugins
 {
     public class PluginService
     {
         private readonly IOptions<DataDirectories> _dataDirectories;
+        private readonly IMemoryCache _memoryCache;
+        private readonly ISettingsRepository _settingsRepository;
         private readonly BTCPayServerOptions _btcPayServerOptions;
         private readonly HttpClient _githubClient;
-        public PluginService(IEnumerable<IBTCPayServerPlugin> btcPayServerPlugins,
+        public PluginService(
-            IHttpClientFactory httpClientFactory, BTCPayServerOptions btcPayServerOptions, IOptions<DataDirectories> dataDirectories)
+            ISettingsRepository settingsRepository,
+            IEnumerable<IBTCPayServerPlugin> btcPayServerPlugins,
+            IHttpClientFactory httpClientFactory, BTCPayServerOptions btcPayServerOptions, 
+            IOptions<DataDirectories> dataDirectories, IMemoryCache memoryCache)
         {
             LoadedPlugins = btcPayServerPlugins;
             _githubClient = httpClientFactory.CreateClient();
             _githubClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("btcpayserver", "1"));
+            _settingsRepository = settingsRepository;
             _btcPayServerOptions = btcPayServerOptions;
             _dataDirectories = dataDirectories;
+            _memoryCache = memoryCache;
+        }
+
+        private async Task<string> CallHttpAndCache(string uri)
+        {
+            var cacheTime = TimeSpan.FromMinutes(30);
+            return await _memoryCache.GetOrCreateAsync(nameof(PluginService) + uri, async entry =>
+            {
+                entry.AbsoluteExpiration = DateTimeOffset.UtcNow + cacheTime;
+                return await  _githubClient.GetStringAsync(uri);
+            });
         }
 
         public IEnumerable<IBTCPayServerPlugin> LoadedPlugins { get; }
 
-        public async Task<AvailablePlugin[]> GetRemotePlugins(string path = "")
+        public async Task<AvailablePlugin[]> GetRemotePlugins()
         {
-            var resp = await _githubClient
+            var resp = await CallHttpAndCache($"https://api.github.com/repos/{_btcPayServerOptions.PluginRemote}/git/trees/master?recursive=1");
-                .GetStringAsync(new Uri($"https://api.github.com/repos/{_btcPayServerOptions.PluginRemote}/contents/{path}"));
-            var files = JsonConvert.DeserializeObject<GithubFile[]>(resp);
-            var dirs = files.Where(file => file.Type == "dir");
-            var result = dirs.Select(file => GetRemotePlugins(file.Path));
 
-            var fileTask = Task.WhenAll(files
+            var respObj = JObject.Parse(resp)["tree"] as JArray;
-                .Where(file => file.Type == "file" && file.Name.EndsWith($"{PluginManager.BTCPayPluginSuffix}.json",
+
-                    StringComparison.InvariantCulture)).Select(async file =>
+            var detectedPlugins = respObj.Where(token => token["path"].ToString().EndsWith(".btcpay"));
+
+            List<Task<AvailablePlugin>> result = new List<Task<AvailablePlugin>>();
+            foreach (JToken detectedPlugin in detectedPlugins)
+            {
+                var pluginName = detectedPlugin["path"].ToString();
+                
+                var metadata =  respObj.SingleOrDefault(token => (pluginName + ".json")== token["path"].ToString());
+                if (metadata is null)
                 {
-                    return await _githubClient.GetStringAsync(file.DownloadUrl).ContinueWith(
+                    continue;
-                        task =>
+                }
-                        {
+                result.Add( CallHttpAndCache(metadata["url"].ToString())
-                            var r = JsonConvert.DeserializeObject<AvailablePlugin>(task.Result);
+                    .ContinueWith(
-                            r.Path = path;
+                    task =>
-                            return r;
+                    {
-                        }, TaskScheduler.Current);
+                        var d = JObject.Parse(task.Result);
-                }));
+
-            return (await Task.WhenAll( result.Concat(new[] { fileTask })).ContinueWith(task => task.Result.SelectMany(plugins => plugins))).ToArray();
+                        var content = Encoders.Base64.DecodeData(d["content"].Value<string>());
+
+                        var r = JsonConvert.DeserializeObject<AvailablePlugin>(Encoding.UTF8.GetString(content));
+                        r.Path = $"https://raw.githubusercontent.com/{_btcPayServerOptions.PluginRemote}/master/{pluginName}";
+                        return r;
+                    }, TaskScheduler.Current));
+                
+            }
+
+            return await Task.WhenAll(result);
         }
-        
-        
-
         public async Task DownloadRemotePlugin(string plugin, string path)
         {
             var dest = _dataDirectories.Value.PluginDir;
-            var resp = await _githubClient
-                .GetStringAsync(new Uri($"https://api.github.com/repos/{_btcPayServerOptions.PluginRemote}/contents/{path}"));
-            var files = JsonConvert.DeserializeObject<GithubFile[]>(resp);
-            var ext = files.SingleOrDefault(file => file.Name == $"{plugin}{PluginManager.BTCPayPluginSuffix}");
-            if (ext is null)
-            {
-                throw new Exception("Plugin not found on remote");
-            }
             
-            if (!ext.Name.IsValidFileName())
+            var filedest = Path.Join(dest, plugin);
-            {
-                throw new Exception("Invalid file name");
-            }
-            var filedest = Path.Join(dest, ext.Name);
             Directory.CreateDirectory(Path.GetDirectoryName(filedest));
-            using var resp2 = await _githubClient.GetAsync(ext.DownloadUrl);
+            using var resp2 = await _githubClient.GetAsync(path); 
             using var fs = new FileStream(filedest, FileMode.Create, FileAccess.ReadWrite);
             await resp2.Content.CopyToAsync(fs);
             await fs.FlushAsync();