From 344755cbd03a8da7ace6b6d63d5d0ba5c0436f1d Mon Sep 17 00:00:00 2001 From: Kukks Date: Wed, 6 Mar 2019 12:33:48 +0100 Subject: [PATCH 1/6] Add policy to discourage search engines + build robots.txt dynamcally closes #390 --- BTCPayServer/BTCPayServer.csproj | 1 + BTCPayServer/Hosting/BTCPayRobotProvider.cs | 38 +++++++++++++++++++++ BTCPayServer/Hosting/Startup.cs | 5 +-- BTCPayServer/Services/PoliciesSettings.cs | 4 +++ BTCPayServer/Views/Server/Policies.cshtml | 4 +++ 5 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 BTCPayServer/Hosting/BTCPayRobotProvider.cs diff --git a/BTCPayServer/BTCPayServer.csproj b/BTCPayServer/BTCPayServer.csproj index 7aa659748..0602925c6 100644 --- a/BTCPayServer/BTCPayServer.csproj +++ b/BTCPayServer/BTCPayServer.csproj @@ -55,6 +55,7 @@ + diff --git a/BTCPayServer/Hosting/BTCPayRobotProvider.cs b/BTCPayServer/Hosting/BTCPayRobotProvider.cs new file mode 100644 index 000000000..f3c12eff2 --- /dev/null +++ b/BTCPayServer/Hosting/BTCPayRobotProvider.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using BTCPayServer.Services; +using Robotify.AspNetCore; + +namespace BTCPayServer.Hosting +{ + public class BTCPayRobotProvider : IRobotifyRobotGroupProvider + { + private readonly SettingsRepository _SettingsRepository; + + public BTCPayRobotProvider(SettingsRepository settingsRepository) + { + _SettingsRepository = settingsRepository; + } + public IEnumerable Get() + { + var settings = _SettingsRepository.GetSettingAsync().GetAwaiter().GetResult(); + if (settings.DiscourageSearchEngines) + { + yield return new RobotGroup() + { + UserAgent = "*", + Disallow = new[] {"/"} + }; + } + else + { + yield return new RobotGroup() + { + UserAgent = "*", + Disallow = Array.Empty() + }; + } + + } + } +} \ No newline at end of file diff --git a/BTCPayServer/Hosting/Startup.cs b/BTCPayServer/Hosting/Startup.cs index 39aeb4994..cdecaad68 100644 --- a/BTCPayServer/Hosting/Startup.cs +++ b/BTCPayServer/Hosting/Startup.cs @@ -3,7 +3,6 @@ using System.Reflection; using System.Linq; using Microsoft.AspNetCore.Builder; using System; -using System.Collections.Generic; using System.Text; using Microsoft.Extensions.DependencyInjection; @@ -14,7 +13,6 @@ using BTCPayServer.Authentication; using Microsoft.EntityFrameworkCore; using BTCPayServer.Filters; using Microsoft.AspNetCore.Mvc.Infrastructure; -using BTCPayServer.Services; using BTCPayServer.Models; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.HttpOverrides; @@ -39,6 +37,7 @@ using BTCPayServer.PaymentRequest; using Meziantou.AspNetCore.BundleTagHelpers; using BTCPayServer.Security; using BTCPayServer.Services.Apps; +using Robotify.AspNetCore; namespace BTCPayServer.Hosting { @@ -62,6 +61,7 @@ namespace BTCPayServer.Hosting Logs.Configure(LoggerFactory); services.ConfigureBTCPayServer(Configuration); services.AddMemoryCache(); + services.AddRobotify(configurer => configurer.AddRobotGroupProvider()); services.AddIdentity() .AddEntityFrameworkStores() .AddDefaultTokenProviders(); @@ -179,6 +179,7 @@ namespace BTCPayServer.Hosting }); app.UseWebSockets(); app.UseStatusCodePages(); + app.UseRobotify(); app.UseMvc(routes => { routes.MapRoute( diff --git a/BTCPayServer/Services/PoliciesSettings.cs b/BTCPayServer/Services/PoliciesSettings.cs index b1ef0a423..cc2766483 100644 --- a/BTCPayServer/Services/PoliciesSettings.cs +++ b/BTCPayServer/Services/PoliciesSettings.cs @@ -18,5 +18,9 @@ namespace BTCPayServer.Services [JsonProperty(DefaultValueHandling = DefaultValueHandling.Populate)] [Display(Name = "Disable registration")] public bool LockSubscription { get; set; } + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Populate)] + [Display(Name = "Discourage search engines from indexing this site")] + public bool DiscourageSearchEngines { get; set; } } } diff --git a/BTCPayServer/Views/Server/Policies.cshtml b/BTCPayServer/Views/Server/Policies.cshtml index 69f072da2..85af6780c 100644 --- a/BTCPayServer/Views/Server/Policies.cshtml +++ b/BTCPayServer/Views/Server/Policies.cshtml @@ -21,6 +21,10 @@
+
+
+ +
From 8b6c4a938381dc925a4666a6d1fa9fc58dd75bcb Mon Sep 17 00:00:00 2001 From: Kukks Date: Sat, 9 Mar 2019 14:23:55 +0100 Subject: [PATCH 2/6] simplifed robots generator --- BTCPayServer/BTCPayServer.csproj | 1 - BTCPayServer/Controllers/HomeController.cs | 17 ++++++++- BTCPayServer/Hosting/BTCPayRobotProvider.cs | 38 --------------------- BTCPayServer/Hosting/Startup.cs | 2 -- 4 files changed, 16 insertions(+), 42 deletions(-) delete mode 100644 BTCPayServer/Hosting/BTCPayRobotProvider.cs diff --git a/BTCPayServer/BTCPayServer.csproj b/BTCPayServer/BTCPayServer.csproj index 0602925c6..7aa659748 100644 --- a/BTCPayServer/BTCPayServer.csproj +++ b/BTCPayServer/BTCPayServer.csproj @@ -55,7 +55,6 @@ - diff --git a/BTCPayServer/Controllers/HomeController.cs b/BTCPayServer/Controllers/HomeController.cs index 6747570ff..5c46284e0 100644 --- a/BTCPayServer/Controllers/HomeController.cs +++ b/BTCPayServer/Controllers/HomeController.cs @@ -8,6 +8,8 @@ using BTCPayServer.Models; using NBitcoin.DataEncoders; using NBitcoin.Payment; using System.Net.Http; +using System.Text; +using BTCPayServer.Services; using Newtonsoft.Json.Linq; using NBitcoin; using Newtonsoft.Json; @@ -16,10 +18,12 @@ namespace BTCPayServer.Controllers { public class HomeController : Controller { + private readonly SettingsRepository _SettingsRepository; public IHttpClientFactory HttpClientFactory { get; } - public HomeController(IHttpClientFactory httpClientFactory) + public HomeController(IHttpClientFactory httpClientFactory, SettingsRepository settingsRepository) { + _SettingsRepository = settingsRepository; HttpClientFactory = httpClientFactory; } public IActionResult Index() @@ -103,5 +107,16 @@ namespace BTCPayServer.Controllers { return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); } + [Route("robots.txt")] + public async Task Robots() + { + var settings = await _SettingsRepository.GetSettingAsync(); + StringBuilder stringBuilder = new StringBuilder(); + + stringBuilder.AppendLine("user-agent: *"); + stringBuilder.AppendLine($"disallow: {(settings.DiscourageSearchEngines? "/": string.Empty)}"); + + return Content(stringBuilder.ToString(), "text/plain", Encoding.UTF8); + } } } diff --git a/BTCPayServer/Hosting/BTCPayRobotProvider.cs b/BTCPayServer/Hosting/BTCPayRobotProvider.cs deleted file mode 100644 index f3c12eff2..000000000 --- a/BTCPayServer/Hosting/BTCPayRobotProvider.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Collections.Generic; -using BTCPayServer.Services; -using Robotify.AspNetCore; - -namespace BTCPayServer.Hosting -{ - public class BTCPayRobotProvider : IRobotifyRobotGroupProvider - { - private readonly SettingsRepository _SettingsRepository; - - public BTCPayRobotProvider(SettingsRepository settingsRepository) - { - _SettingsRepository = settingsRepository; - } - public IEnumerable Get() - { - var settings = _SettingsRepository.GetSettingAsync().GetAwaiter().GetResult(); - if (settings.DiscourageSearchEngines) - { - yield return new RobotGroup() - { - UserAgent = "*", - Disallow = new[] {"/"} - }; - } - else - { - yield return new RobotGroup() - { - UserAgent = "*", - Disallow = Array.Empty() - }; - } - - } - } -} \ No newline at end of file diff --git a/BTCPayServer/Hosting/Startup.cs b/BTCPayServer/Hosting/Startup.cs index cdecaad68..23e04db8e 100644 --- a/BTCPayServer/Hosting/Startup.cs +++ b/BTCPayServer/Hosting/Startup.cs @@ -61,7 +61,6 @@ namespace BTCPayServer.Hosting Logs.Configure(LoggerFactory); services.ConfigureBTCPayServer(Configuration); services.AddMemoryCache(); - services.AddRobotify(configurer => configurer.AddRobotGroupProvider()); services.AddIdentity() .AddEntityFrameworkStores() .AddDefaultTokenProviders(); @@ -179,7 +178,6 @@ namespace BTCPayServer.Hosting }); app.UseWebSockets(); app.UseStatusCodePages(); - app.UseRobotify(); app.UseMvc(routes => { routes.MapRoute( From 6746a5cbd55bd4396cf9cf88c82b0bdd7d65e25b Mon Sep 17 00:00:00 2001 From: Kukks Date: Sat, 9 Mar 2019 14:55:37 +0100 Subject: [PATCH 3/6] add meta for noindex,nofollow if policy set --- .../Components/RobotsMetaViewComponent.cs | 29 +++++++++++++++++++ BTCPayServer/Hosting/Startup.cs | 1 - BTCPayServer/Models/RobotsMetaViewModel.cs | 7 +++++ .../Components/RobotsMeta/Default.cshtml | 5 ++++ BTCPayServer/Views/Shared/_Layout.cshtml | 3 ++ 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 BTCPayServer/Components/RobotsMetaViewComponent.cs create mode 100644 BTCPayServer/Models/RobotsMetaViewModel.cs create mode 100644 BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml diff --git a/BTCPayServer/Components/RobotsMetaViewComponent.cs b/BTCPayServer/Components/RobotsMetaViewComponent.cs new file mode 100644 index 000000000..b44310646 --- /dev/null +++ b/BTCPayServer/Components/RobotsMetaViewComponent.cs @@ -0,0 +1,29 @@ +using System.Threading.Tasks; +using BTCPayServer.Models; +using BTCPayServer.Services; +using Microsoft.AspNetCore.Mvc; + +namespace BTCPayServer.Components +{ + public class RobotsMetaViewComponent : ViewComponent + { + private readonly SettingsRepository _SettingsRepository; + + public RobotsMetaViewComponent(SettingsRepository settingsRepository) + { + _SettingsRepository = settingsRepository; + } + + public async Task InvokeAsync() + { + var policies = await _SettingsRepository.GetSettingAsync(); + + return View(new RobotsMetaViewModel() + { + DiscourageSearchEngines = policies.DiscourageSearchEngines + }); + } + } + + +} diff --git a/BTCPayServer/Hosting/Startup.cs b/BTCPayServer/Hosting/Startup.cs index 23e04db8e..8f7cf3f30 100644 --- a/BTCPayServer/Hosting/Startup.cs +++ b/BTCPayServer/Hosting/Startup.cs @@ -37,7 +37,6 @@ using BTCPayServer.PaymentRequest; using Meziantou.AspNetCore.BundleTagHelpers; using BTCPayServer.Security; using BTCPayServer.Services.Apps; -using Robotify.AspNetCore; namespace BTCPayServer.Hosting { diff --git a/BTCPayServer/Models/RobotsMetaViewModel.cs b/BTCPayServer/Models/RobotsMetaViewModel.cs new file mode 100644 index 000000000..609c18c4f --- /dev/null +++ b/BTCPayServer/Models/RobotsMetaViewModel.cs @@ -0,0 +1,7 @@ +namespace BTCPayServer.Models +{ + public class RobotsMetaViewModel + { + public bool DiscourageSearchEngines { get; set; } + } +} \ No newline at end of file diff --git a/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml b/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml new file mode 100644 index 000000000..62c824110 --- /dev/null +++ b/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml @@ -0,0 +1,5 @@ +@model BTCPayServer.Models.RobotsMetaViewModel +@if (Model.DiscourageSearchEngines) +{ + +} diff --git a/BTCPayServer/Views/Shared/_Layout.cshtml b/BTCPayServer/Views/Shared/_Layout.cshtml index 2f2a54e04..d16a920ba 100644 --- a/BTCPayServer/Views/Shared/_Layout.cshtml +++ b/BTCPayServer/Views/Shared/_Layout.cshtml @@ -15,6 +15,9 @@ + + @await Component.InvokeAsync("RobotsMeta", new { }) + BTCPay Server From 1888e4fe2b337c0b7cf7e2627b8ac532190bb5de Mon Sep 17 00:00:00 2001 From: Kukks Date: Sat, 9 Mar 2019 15:01:56 +0100 Subject: [PATCH 4/6] remove robots and remove nofollow --- BTCPayServer/Controllers/HomeController.cs | 20 +------------------ .../Components/RobotsMeta/Default.cshtml | 2 +- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/BTCPayServer/Controllers/HomeController.cs b/BTCPayServer/Controllers/HomeController.cs index 5c46284e0..01dcccbee 100644 --- a/BTCPayServer/Controllers/HomeController.cs +++ b/BTCPayServer/Controllers/HomeController.cs @@ -1,15 +1,10 @@ using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using BTCPayServer.Models; -using NBitcoin.DataEncoders; using NBitcoin.Payment; using System.Net.Http; -using System.Text; -using BTCPayServer.Services; using Newtonsoft.Json.Linq; using NBitcoin; using Newtonsoft.Json; @@ -18,12 +13,10 @@ namespace BTCPayServer.Controllers { public class HomeController : Controller { - private readonly SettingsRepository _SettingsRepository; public IHttpClientFactory HttpClientFactory { get; } - public HomeController(IHttpClientFactory httpClientFactory, SettingsRepository settingsRepository) + public HomeController(IHttpClientFactory httpClientFactory) { - _SettingsRepository = settingsRepository; HttpClientFactory = httpClientFactory; } public IActionResult Index() @@ -107,16 +100,5 @@ namespace BTCPayServer.Controllers { return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); } - [Route("robots.txt")] - public async Task Robots() - { - var settings = await _SettingsRepository.GetSettingAsync(); - StringBuilder stringBuilder = new StringBuilder(); - - stringBuilder.AppendLine("user-agent: *"); - stringBuilder.AppendLine($"disallow: {(settings.DiscourageSearchEngines? "/": string.Empty)}"); - - return Content(stringBuilder.ToString(), "text/plain", Encoding.UTF8); - } } } diff --git a/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml b/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml index 62c824110..9194de319 100644 --- a/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml +++ b/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml @@ -1,5 +1,5 @@ @model BTCPayServer.Models.RobotsMetaViewModel @if (Model.DiscourageSearchEngines) { - + } From 053c2da9f1e74af80ef224a4e733f2d954207c91 Mon Sep 17 00:00:00 2001 From: Kukks Date: Sat, 9 Mar 2019 16:29:04 +0100 Subject: [PATCH 5/6] use thememanager instead of view component --- .../Components/RobotsMetaViewComponent.cs | 29 ------------------- .../HostedServices/CssThemeManager.cs | 2 ++ .../Components/RobotsMeta/Default.cshtml | 5 ---- BTCPayServer/Views/Shared/_Layout.cshtml | 11 +++---- 4 files changed, 8 insertions(+), 39 deletions(-) delete mode 100644 BTCPayServer/Components/RobotsMetaViewComponent.cs delete mode 100644 BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml diff --git a/BTCPayServer/Components/RobotsMetaViewComponent.cs b/BTCPayServer/Components/RobotsMetaViewComponent.cs deleted file mode 100644 index b44310646..000000000 --- a/BTCPayServer/Components/RobotsMetaViewComponent.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System.Threading.Tasks; -using BTCPayServer.Models; -using BTCPayServer.Services; -using Microsoft.AspNetCore.Mvc; - -namespace BTCPayServer.Components -{ - public class RobotsMetaViewComponent : ViewComponent - { - private readonly SettingsRepository _SettingsRepository; - - public RobotsMetaViewComponent(SettingsRepository settingsRepository) - { - _SettingsRepository = settingsRepository; - } - - public async Task InvokeAsync() - { - var policies = await _SettingsRepository.GetSettingAsync(); - - return View(new RobotsMetaViewModel() - { - DiscourageSearchEngines = policies.DiscourageSearchEngines - }); - } - } - - -} diff --git a/BTCPayServer/HostedServices/CssThemeManager.cs b/BTCPayServer/HostedServices/CssThemeManager.cs index 1e240c940..6b2c6b139 100644 --- a/BTCPayServer/HostedServices/CssThemeManager.cs +++ b/BTCPayServer/HostedServices/CssThemeManager.cs @@ -45,10 +45,12 @@ namespace BTCPayServer.HostedServices } public bool ShowRegister { get; set; } + public bool DiscourageSearchEngines { get; set; } internal void Update(PoliciesSettings data) { ShowRegister = !data.LockSubscription; + DiscourageSearchEngines = data.DiscourageSearchEngines; } } diff --git a/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml b/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml deleted file mode 100644 index 9194de319..000000000 --- a/BTCPayServer/Views/Shared/Components/RobotsMeta/Default.cshtml +++ /dev/null @@ -1,5 +0,0 @@ -@model BTCPayServer.Models.RobotsMetaViewModel -@if (Model.DiscourageSearchEngines) -{ - -} diff --git a/BTCPayServer/Views/Shared/_Layout.cshtml b/BTCPayServer/Views/Shared/_Layout.cshtml index d16a920ba..2f963dd94 100644 --- a/BTCPayServer/Views/Shared/_Layout.cshtml +++ b/BTCPayServer/Views/Shared/_Layout.cshtml @@ -15,9 +15,10 @@ - - @await Component.InvokeAsync("RobotsMeta", new { }) - + @if (themeManager.DiscourageSearchEngines) + { + + } BTCPay Server @@ -61,7 +62,7 @@
    @if (SignInManager.IsSignedIn(User)) { - @if (User.IsInRole(Roles.ServerAdmin)) + if (User.IsInRole(Roles.ServerAdmin)) {
  • Server settings
    • } @@ -78,7 +79,7 @@ } else { - @if (themeManager.ShowRegister) + if (themeManager.ShowRegister) {
  • Register
    • } From 4deb853914011eef125c97e097a10bfa29b890b1 Mon Sep 17 00:00:00 2001 From: Kukks Date: Sat, 9 Mar 2019 16:35:53 +0100 Subject: [PATCH 6/6] remo useless mode now --- BTCPayServer/Models/RobotsMetaViewModel.cs | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 BTCPayServer/Models/RobotsMetaViewModel.cs diff --git a/BTCPayServer/Models/RobotsMetaViewModel.cs b/BTCPayServer/Models/RobotsMetaViewModel.cs deleted file mode 100644 index 609c18c4f..000000000 --- a/BTCPayServer/Models/RobotsMetaViewModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BTCPayServer.Models -{ - public class RobotsMetaViewModel - { - public bool DiscourageSearchEngines { get; set; } - } -} \ No newline at end of file