From d86cc9192e97a3044562b900c0ee6744ba49938d Mon Sep 17 00:00:00 2001 From: Andrew Camilleri Date: Fri, 24 May 2019 06:44:23 +0000 Subject: [PATCH] Support temporary links for local file system provider (#848) * wip * Support temporary links for local file system provider * pass base url to file services * fix test * do not crash on errors with local filesystem * remove console * fix paranthesis --- BTCPayServer.Tests/StorageTests.cs | 45 ++++++++--- .../Controllers/ServerController.Storage.cs | 5 +- BTCPayServer/Controllers/StorageController.cs | 11 ++- BTCPayServer/Extensions.cs | 5 ++ BTCPayServer/Storage/Services/FileService.cs | 9 ++- ...ntyTwentyStorageFileProviderServiceBase.cs | 5 +- .../FileSystemFileProviderService.cs | 58 ++++++++------ .../TemporaryLocalFileDescriptor.cs | 11 +++ .../TemporaryLocalFileProvider.cs | 53 ++++++++++++ .../Providers/IStorageProviderService.cs | 4 +- BTCPayServer/Storage/StorageExtensions.cs | 80 +++++++++++++------ 11 files changed, 214 insertions(+), 72 deletions(-) create mode 100644 BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileDescriptor.cs create mode 100644 BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileProvider.cs diff --git a/BTCPayServer.Tests/StorageTests.cs b/BTCPayServer.Tests/StorageTests.cs index 1f53caee6..b2f07dc7f 100644 --- a/BTCPayServer.Tests/StorageTests.cs +++ b/BTCPayServer.Tests/StorageTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using BTCPayServer.Controllers; @@ -9,6 +10,7 @@ using BTCPayServer.Storage.Services.Providers.AzureBlobStorage.Configuration; using BTCPayServer.Storage.Services.Providers.FileSystemStorage.Configuration; using BTCPayServer.Storage.ViewModels; using BTCPayServer.Tests.Logging; +using DBriize.Utils; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Internal; using Microsoft.AspNetCore.Mvc; @@ -38,15 +40,6 @@ namespace BTCPayServer.Tests user.GrantAccess(); var controller = tester.PayTester.GetController(user.UserId, user.StoreId); -// //For some reason, the tests cache something on circleci and this is set by default -// //Initially, there is no configuration, make sure we display the choices available to configure -// Assert.IsType(Assert.IsType(await controller.Storage()).Model); -// -// //the file list should tell us it's not configured: -// var viewFilesViewModelInitial = -// Assert.IsType(Assert.IsType(await controller.Files()).Model); -// Assert.False(viewFilesViewModelInitial.StorageConfigured); - //Once we select a provider, redirect to its view var localResult = Assert @@ -196,6 +189,7 @@ namespace BTCPayServer.Tests var fileId = uploadFormFileResult.RouteValues["fileId"].ToString(); Assert.Equal("Files", uploadFormFileResult.ActionName); + //check if file was uploaded and saved in db var viewFilesViewModel = Assert.IsType(Assert.IsType(await controller.Files(fileId)).Model); @@ -203,21 +197,48 @@ namespace BTCPayServer.Tests Assert.Equal(fileId, viewFilesViewModel.SelectedFileId); Assert.NotEmpty(viewFilesViewModel.DirectFileUrl); - + + //verify file is available and the same var net = new System.Net.WebClient(); var data = await net.DownloadStringTaskAsync(new Uri(viewFilesViewModel.DirectFileUrl)); Assert.Equal(fileContent, data); - + + //create a temporary link to file + var tmpLinkGenerate = Assert.IsType(await controller.CreateTemporaryFileUrl(fileId, + new ServerController.CreateTemporaryFileUrlViewModel() + { + IsDownload = true, + TimeAmount = 1, + TimeType = ServerController.CreateTemporaryFileUrlViewModel.TmpFileTimeType.Minutes + })); + Assert.True(tmpLinkGenerate.RouteValues.ContainsKey("StatusMessage")); + var statusMessageModel = new StatusMessageModel(tmpLinkGenerate.RouteValues["StatusMessage"].ToString()); + Assert.Equal(StatusMessageModel.StatusSeverity.Success, statusMessageModel.Severity); + var index = statusMessageModel.Html.IndexOf("target='_blank'>"); + var url = statusMessageModel.Html.Substring(index).ReplaceMultiple(new Dictionary() + { + {"", string.Empty}, {"target='_blank'>", string.Empty} + }); + //verify tmpfile is available and the same + data = await net.DownloadStringTaskAsync(new Uri(url)); + Assert.Equal(fileContent, data); + + + //delete file Assert.Equal(StatusMessageModel.StatusSeverity.Success, new StatusMessageModel(Assert .IsType(await controller.DeleteFile(fileId)) .RouteValues["statusMessage"].ToString()).Severity); - + + //attempt to fetch deleted file viewFilesViewModel = Assert.IsType(Assert.IsType(await controller.Files(fileId)).Model); Assert.Null(viewFilesViewModel.DirectFileUrl); Assert.Null(viewFilesViewModel.SelectedFileId); } + + + diff --git a/BTCPayServer/Controllers/ServerController.Storage.cs b/BTCPayServer/Controllers/ServerController.Storage.cs index 7b96661c1..5d8e04f67 100644 --- a/BTCPayServer/Controllers/ServerController.Storage.cs +++ b/BTCPayServer/Controllers/ServerController.Storage.cs @@ -27,7 +27,7 @@ namespace BTCPayServer.Controllers public async Task Files(string fileId = null, string statusMessage = null) { TempData["StatusMessage"] = statusMessage; - var fileUrl = string.IsNullOrEmpty(fileId) ? null : await _FileService.GetFileUrl(fileId); + var fileUrl = string.IsNullOrEmpty(fileId) ? null : await _FileService.GetFileUrl(Request.GetAbsoluteRootUri(), fileId); return View(new ViewFilesViewModel() { @@ -116,12 +116,13 @@ namespace BTCPayServer.Controllers throw new ArgumentOutOfRangeException(); } - var url = await _FileService.GetTemporaryFileUrl(fileId, expiry, viewModel.IsDownload); + var url = await _FileService.GetTemporaryFileUrl(Request.GetAbsoluteRootUri(), fileId, expiry, viewModel.IsDownload); return RedirectToAction(nameof(Files), new { StatusMessage = new StatusMessageModel() { + Severity = StatusMessageModel.StatusSeverity.Success, Html = $"Generated Temporary Url for file {file.FileName} which expires at {expiry.ToBrowserDate()}. {url}" }.ToString(), diff --git a/BTCPayServer/Controllers/StorageController.cs b/BTCPayServer/Controllers/StorageController.cs index 4bac3beac..8c2a77978 100644 --- a/BTCPayServer/Controllers/StorageController.cs +++ b/BTCPayServer/Controllers/StorageController.cs @@ -1,23 +1,28 @@ +using System; using System.Threading.Tasks; +using BTCPayServer.Configuration; using BTCPayServer.Storage.Services; +using BTCPayServer.Storage.Services.Providers.FileSystemStorage; using Microsoft.AspNetCore.Mvc; namespace BTCPayServer.Storage { [Route("Storage")] - public class StorageController + public class StorageController : Controller { private readonly FileService _FileService; + private string _dir; - public StorageController(FileService fileService) + public StorageController(FileService fileService, BTCPayServerOptions serverOptions) { _FileService = fileService; + _dir =FileSystemFileProviderService.GetTempStorageDir(serverOptions); } [HttpGet("{fileId}")] public async Task GetFile(string fileId) { - var url = await _FileService.GetFileUrl(fileId); + var url = await _FileService.GetFileUrl(Request.GetAbsoluteRootUri(), fileId); return new RedirectResult(url); } } diff --git a/BTCPayServer/Extensions.cs b/BTCPayServer/Extensions.cs index 73a1feae6..e36c66359 100644 --- a/BTCPayServer/Extensions.cs +++ b/BTCPayServer/Extensions.cs @@ -211,6 +211,11 @@ namespace BTCPayServer request.PathBase.ToUriComponent()); } + public static Uri GetAbsoluteRootUri(this HttpRequest request) + { + return new Uri(request.GetAbsoluteRoot()); + } + public static string GetCurrentUrl(this HttpRequest request) { return string.Concat( diff --git a/BTCPayServer/Storage/Services/FileService.cs b/BTCPayServer/Storage/Services/FileService.cs index 3e5e1d95a..b2ba7c467 100644 --- a/BTCPayServer/Storage/Services/FileService.cs +++ b/BTCPayServer/Storage/Services/FileService.cs @@ -34,20 +34,21 @@ namespace BTCPayServer.Storage.Services return storedFile; } - public async Task GetFileUrl(string fileId) + public async Task GetFileUrl(Uri baseUri, string fileId) { var settings = await _SettingsRepository.GetSettingAsync(); var provider = GetProvider(settings); var storedFile = await _FileRepository.GetFile(fileId); - return storedFile == null ? null: await provider.GetFileUrl(storedFile, settings); + return storedFile == null ? null: await provider.GetFileUrl(baseUri, storedFile, settings); } - public async Task GetTemporaryFileUrl(string fileId, DateTimeOffset expiry, bool isDownload) + public async Task GetTemporaryFileUrl(Uri baseUri, string fileId, DateTimeOffset expiry, + bool isDownload) { var settings = await _SettingsRepository.GetSettingAsync(); var provider = GetProvider(settings); var storedFile = await _FileRepository.GetFile(fileId); - return storedFile == null ? null: await provider.GetTemporaryFileUrl(storedFile, settings,expiry,isDownload); + return storedFile == null ? null: await provider.GetTemporaryFileUrl(baseUri, storedFile, settings,expiry,isDownload); } public async Task RemoveFile(string fileId, string userId) diff --git a/BTCPayServer/Storage/Services/Providers/BaseTwentyTwentyStorageFileProviderServiceBase.cs b/BTCPayServer/Storage/Services/Providers/BaseTwentyTwentyStorageFileProviderServiceBase.cs index 5f326b540..03b5551ab 100644 --- a/BTCPayServer/Storage/Services/Providers/BaseTwentyTwentyStorageFileProviderServiceBase.cs +++ b/BTCPayServer/Storage/Services/Providers/BaseTwentyTwentyStorageFileProviderServiceBase.cs @@ -38,14 +38,15 @@ namespace BTCPayServer.Storage.Services.Providers }; } - public virtual async Task GetFileUrl(StoredFile storedFile, StorageSettings configuration) + public virtual async Task GetFileUrl(Uri baseUri, StoredFile storedFile, StorageSettings configuration) { var providerConfiguration = GetProviderConfiguration(configuration); var provider = await GetStorageProvider(providerConfiguration); return provider.GetBlobUrl(providerConfiguration.ContainerName, storedFile.StorageFileName); } - public virtual async Task GetTemporaryFileUrl(StoredFile storedFile, StorageSettings configuration, + public virtual async Task GetTemporaryFileUrl(Uri baseUri, StoredFile storedFile, + StorageSettings configuration, DateTimeOffset expiry, bool isDownload, BlobUrlAccess access = BlobUrlAccess.Read) { var providerConfiguration = GetProviderConfiguration(configuration); diff --git a/BTCPayServer/Storage/Services/Providers/FileSystemStorage/FileSystemFileProviderService.cs b/BTCPayServer/Storage/Services/Providers/FileSystemStorage/FileSystemFileProviderService.cs index 62d47bbea..b91670869 100644 --- a/BTCPayServer/Storage/Services/Providers/FileSystemStorage/FileSystemFileProviderService.cs +++ b/BTCPayServer/Storage/Services/Providers/FileSystemStorage/FileSystemFileProviderService.cs @@ -2,11 +2,11 @@ using System; using System.IO; using System.Threading.Tasks; using BTCPayServer.Configuration; -using BTCPayServer.Services; using BTCPayServer.Storage.Models; using BTCPayServer.Storage.Services.Providers.FileSystemStorage.Configuration; +using ExchangeSharp; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; +using Newtonsoft.Json; using TwentyTwenty.Storage; using TwentyTwenty.Storage.Local; @@ -15,16 +15,11 @@ namespace BTCPayServer.Storage.Services.Providers.FileSystemStorage public class FileSystemFileProviderService : BaseTwentyTwentyStorageFileProviderServiceBase { - private readonly BTCPayServerEnvironment _BtcPayServerEnvironment; - private readonly BTCPayServerOptions _Options; - private readonly IHttpContextAccessor _HttpContextAccessor; + private readonly BTCPayServerOptions _options; - public FileSystemFileProviderService(BTCPayServerEnvironment btcPayServerEnvironment, - BTCPayServerOptions options, IHttpContextAccessor httpContextAccessor) + public FileSystemFileProviderService(BTCPayServerOptions options) { - _BtcPayServerEnvironment = btcPayServerEnvironment; - _Options = options; - _HttpContextAccessor = httpContextAccessor; + _options = options; } public const string LocalStorageDirectoryName = "LocalStorage"; @@ -32,7 +27,12 @@ namespace BTCPayServer.Storage.Services.Providers.FileSystemStorage { return Path.Combine(options.DataDir, LocalStorageDirectoryName); } - + + + public static string GetTempStorageDir(BTCPayServerOptions options) + { + return Path.Combine(GetStorageDir(options), "tmp"); + } public override StorageProvider StorageProvider() { return Storage.Models.StorageProvider.FileSystem; @@ -41,26 +41,38 @@ namespace BTCPayServer.Storage.Services.Providers.FileSystemStorage protected override Task GetStorageProvider(FileSystemStorageConfiguration configuration) { return Task.FromResult( - new LocalStorageProvider(new DirectoryInfo(GetStorageDir(_Options)).FullName)); + new LocalStorageProvider(new DirectoryInfo(GetStorageDir(_options)).FullName)); } - public override async Task GetFileUrl(StoredFile storedFile, StorageSettings configuration) + public override async Task GetFileUrl(Uri baseUri, StoredFile storedFile, StorageSettings configuration) { - var baseResult = await base.GetFileUrl(storedFile, configuration); - var url = - _HttpContextAccessor.HttpContext.Request.IsOnion() - ? _BtcPayServerEnvironment.OnionUrl - : $"{_BtcPayServerEnvironment.ExpectedProtocol}://" + - $"{_BtcPayServerEnvironment.ExpectedHost}" + - $"{_Options.RootPath}{LocalStorageDirectoryName}"; - return baseResult.Replace(new DirectoryInfo(GetStorageDir(_Options)).FullName, url, + var baseResult = await base.GetFileUrl(baseUri, storedFile, configuration); + var url = new Uri(baseUri,LocalStorageDirectoryName ); + return baseResult.Replace(new DirectoryInfo(GetStorageDir(_options)).FullName, url.AbsoluteUri, StringComparison.InvariantCultureIgnoreCase); } - public override async Task GetTemporaryFileUrl(StoredFile storedFile, StorageSettings configuration, DateTimeOffset expiry, bool isDownload, + public override async Task GetTemporaryFileUrl(Uri baseUri, StoredFile storedFile, + StorageSettings configuration, DateTimeOffset expiry, bool isDownload, BlobUrlAccess access = BlobUrlAccess.Read) { - return $"{(await GetFileUrl(storedFile, configuration))}{(isDownload ? "?download" : string.Empty)}"; + + var localFileDescriptor = new TemporaryLocalFileDescriptor() + { + Expiry = expiry, + FileId = storedFile.Id, + IsDownload = isDownload + }; + var name = Guid.NewGuid().ToString(); + var fullPath = Path.Combine(GetTempStorageDir(_options), name); + if (!File.Exists(fullPath)) + { + File.Create(fullPath).Dispose(); + } + + await File.WriteAllTextAsync(Path.Combine(GetTempStorageDir(_options), name), JsonConvert.SerializeObject(localFileDescriptor)); + + return new Uri(baseUri,$"{LocalStorageDirectoryName}tmp/{name}" ).AbsoluteUri; } } } diff --git a/BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileDescriptor.cs b/BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileDescriptor.cs new file mode 100644 index 000000000..df4504d7f --- /dev/null +++ b/BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileDescriptor.cs @@ -0,0 +1,11 @@ +using System; + +namespace BTCPayServer.Storage.Services.Providers.FileSystemStorage +{ + public class TemporaryLocalFileDescriptor + { + public string FileId { get; set; } + public bool IsDownload { get; set; } + public DateTimeOffset Expiry { get; set; } + } +} \ No newline at end of file diff --git a/BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileProvider.cs b/BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileProvider.cs new file mode 100644 index 000000000..bccf4ef37 --- /dev/null +++ b/BTCPayServer/Storage/Services/Providers/FileSystemStorage/TemporaryLocalFileProvider.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.FileProviders.Physical; +using Microsoft.Extensions.Primitives; +using Newtonsoft.Json; + +namespace BTCPayServer.Storage.Services.Providers.FileSystemStorage +{ + public class TemporaryLocalFileProvider : IFileProvider + { + private readonly DirectoryInfo _fileRoot; + private readonly StoredFileRepository _storedFileRepository; + private readonly DirectoryInfo _root; + + public TemporaryLocalFileProvider(DirectoryInfo tmpRoot, DirectoryInfo fileRoot, StoredFileRepository storedFileRepository) + { + _fileRoot = fileRoot; + _storedFileRepository = storedFileRepository; + _root = tmpRoot; + } + public IFileInfo GetFileInfo(string tmpFileId) + { + tmpFileId =tmpFileId.TrimStart('/', '\\'); + var path = Path.Combine(_root.FullName,tmpFileId) ; + if (!File.Exists(path)) + { + return new NotFoundFileInfo(tmpFileId); + } + + var text = File.ReadAllText(path); + var descriptor = JsonConvert.DeserializeObject(text); + if (descriptor.Expiry < DateTime.Now) + { + File.Delete(path); + return new NotFoundFileInfo(tmpFileId); + } + + var storedFile = _storedFileRepository.GetFile(descriptor.FileId).GetAwaiter().GetResult(); + return new PhysicalFileInfo(new FileInfo(Path.Combine(_fileRoot.FullName, storedFile.StorageFileName))); + } + + public IDirectoryContents GetDirectoryContents(string subpath) + { + throw new System.NotImplementedException(); + } + + public IChangeToken Watch(string filter) + { + throw new System.NotImplementedException(); + } + } +} diff --git a/BTCPayServer/Storage/Services/Providers/IStorageProviderService.cs b/BTCPayServer/Storage/Services/Providers/IStorageProviderService.cs index 770678001..72375f395 100644 --- a/BTCPayServer/Storage/Services/Providers/IStorageProviderService.cs +++ b/BTCPayServer/Storage/Services/Providers/IStorageProviderService.cs @@ -10,8 +10,8 @@ namespace BTCPayServer.Storage.Services.Providers { Task AddFile(IFormFile formFile, StorageSettings configuration); Task RemoveFile(StoredFile storedFile, StorageSettings configuration); - Task GetFileUrl(StoredFile storedFile, StorageSettings configuration); - Task GetTemporaryFileUrl(StoredFile storedFile, StorageSettings configuration, + Task GetFileUrl(Uri baseUri, StoredFile storedFile, StorageSettings configuration); + Task GetTemporaryFileUrl(Uri baseUri, StoredFile storedFile, StorageSettings configuration, DateTimeOffset expiry, bool isDownload, BlobUrlAccess access = BlobUrlAccess.Read); StorageProvider StorageProvider(); } diff --git a/BTCPayServer/Storage/StorageExtensions.cs b/BTCPayServer/Storage/StorageExtensions.cs index c81130103..b64c66799 100644 --- a/BTCPayServer/Storage/StorageExtensions.cs +++ b/BTCPayServer/Storage/StorageExtensions.cs @@ -1,15 +1,16 @@ +using System; using System.IO; using BTCPayServer.Configuration; using BTCPayServer.Storage.Services; using BTCPayServer.Storage.Services.Providers; -using BTCPayServer.Storage.Services.Providers.AmazonS3Storage; using BTCPayServer.Storage.Services.Providers.AzureBlobStorage; using BTCPayServer.Storage.Services.Providers.FileSystemStorage; -using BTCPayServer.Storage.Services.Providers.GoogleCloudStorage; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Logging; +using NBitcoin.Logging; namespace BTCPayServer.Storage { @@ -27,31 +28,62 @@ namespace BTCPayServer.Storage public static void UseProviderStorage(this IApplicationBuilder builder, BTCPayServerOptions options) { - var dir = FileSystemFileProviderService.GetStorageDir(options); - - DirectoryInfo dirInfo; - if (!Directory.Exists(dir)) + try { - dirInfo = Directory.CreateDirectory(dir); - } - else - { - dirInfo = new DirectoryInfo(dir); - } - - builder.UseStaticFiles(new StaticFileOptions() - { - ServeUnknownFileTypes = true, - RequestPath = new PathString($"/{FileSystemFileProviderService.LocalStorageDirectoryName}"), - FileProvider = new PhysicalFileProvider(dirInfo.FullName), - OnPrepareResponse = context => + var dir = FileSystemFileProviderService.GetStorageDir(options); + var tmpdir = FileSystemFileProviderService.GetTempStorageDir(options); + DirectoryInfo dirInfo; + if (!Directory.Exists(dir)) { - if (context.Context.Request.Query.ContainsKey("download")) - { - context.Context.Response.Headers["Content-Disposition"] = "attachment"; - } + dirInfo = Directory.CreateDirectory(dir); } - }); + else + { + dirInfo = new DirectoryInfo(dir); + } + + DirectoryInfo tmpdirInfo; + if (!Directory.Exists(tmpdir)) + { + tmpdirInfo = Directory.CreateDirectory(tmpdir); + } + else + { + tmpdirInfo = new DirectoryInfo(tmpdir); + } + + builder.UseStaticFiles(new StaticFileOptions() + { + ServeUnknownFileTypes = true, + RequestPath = new PathString($"/{FileSystemFileProviderService.LocalStorageDirectoryName}"), + FileProvider = new PhysicalFileProvider(dirInfo.FullName), + OnPrepareResponse = context => + { + if (context.Context.Request.Query.ContainsKey("download")) + { + context.Context.Response.Headers["Content-Disposition"] = "attachment"; + } + } + }); + builder.UseStaticFiles(new StaticFileOptions() + { + ServeUnknownFileTypes = true, + RequestPath = new PathString($"/{FileSystemFileProviderService.LocalStorageDirectoryName}tmp"), + FileProvider = new TemporaryLocalFileProvider(tmpdirInfo, dirInfo, + builder.ApplicationServices.GetService()), + OnPrepareResponse = context => + { + if (context.Context.Request.Query.ContainsKey("download")) + { + context.Context.Response.Headers["Content-Disposition"] = "attachment"; + } + } + }); + } + catch (Exception e) + { + Logs.Utils.LogError(e, $"Could not initialize the Local File Storage system(uploading and storing files locally)"); + } } } }