mirror of
https://github.com/aljazceru/btcpayserver.git
synced 2025-12-18 22:44:29 +01:00
Refactor labels (#4179)
* Create new tables * wip * wip * Refactor LegacyLabel * Remove LabelFactory * Add migration * wip * wip * Add pull-payment attachment to tx * Address kukks points
This commit is contained in:
Nicolas Dorier
@@ -2,15 +2,20 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BTCPayServer.Client.Models;
|
||||
using BTCPayServer.Configuration;
|
||||
using BTCPayServer.Data;
|
||||
using BTCPayServer.Logging;
|
||||
using BTCPayServer.Services;
|
||||
using BTCPayServer.Services.Invoices;
|
||||
using BTCPayServer.Services.Labels;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace BTCPayServer.HostedServices
|
||||
{
|
||||
@@ -45,10 +50,235 @@ namespace BTCPayServer.HostedServices
|
||||
{
|
||||
await MigratedInvoiceTextSearchToDb(settings.MigratedInvoiceTextSearchPages ?? 0);
|
||||
}
|
||||
|
||||
// Refresh settings since these operations may run for very long time
|
||||
if (settings.MigratedTransactionLabels != int.MaxValue)
|
||||
{
|
||||
await MigratedTransactionLabels(settings.MigratedTransactionLabels ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#pragma warning disable CS0612 // Type or member is obsolete
|
||||
class LegacyWalletTransactionInfo
|
||||
{
|
||||
public string Comment { get; set; } = string.Empty;
|
||||
[JsonIgnore]
|
||||
public Dictionary<string, LabelData> Labels { get; set; } = new Dictionary<string, LabelData>();
|
||||
}
|
||||
|
||||
static LegacyWalletTransactionInfo GetBlobInfo(WalletTransactionData walletTransactionData)
|
||||
{
|
||||
LegacyWalletTransactionInfo blobInfo;
|
||||
if (walletTransactionData.Blob == null || walletTransactionData.Blob.Length == 0)
|
||||
blobInfo = new LegacyWalletTransactionInfo();
|
||||
else
|
||||
blobInfo = JsonConvert.DeserializeObject<LegacyWalletTransactionInfo>(ZipUtils.Unzip(walletTransactionData.Blob));
|
||||
if (!string.IsNullOrEmpty(walletTransactionData.Labels))
|
||||
{
|
||||
if (walletTransactionData.Labels.StartsWith('['))
|
||||
{
|
||||
foreach (var jtoken in JArray.Parse(walletTransactionData.Labels))
|
||||
{
|
||||
var l = jtoken.Type == JTokenType.String ? Label.Parse(jtoken.Value<string>())
|
||||
: Label.Parse(jtoken.ToString());
|
||||
blobInfo.Labels.TryAdd(l.Text, l);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Legacy path
|
||||
foreach (var token in walletTransactionData.Labels.Split(',',
|
||||
StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var l = Label.Parse(token);
|
||||
blobInfo.Labels.TryAdd(l.Text, l);
|
||||
}
|
||||
}
|
||||
}
|
||||
return blobInfo;
|
||||
}
|
||||
|
||||
internal async Task MigratedTransactionLabels(int startFromOffset)
|
||||
{
|
||||
// Only of 1000, that's what EF does anyway under the hood by default
|
||||
int batchCount = 1000;
|
||||
int total = 0;
|
||||
HashSet<(string WalletId, string LabelId)> existingLabels;
|
||||
using (var db = _dbContextFactory.CreateContext())
|
||||
{
|
||||
total = await db.WalletTransactions.CountAsync();
|
||||
existingLabels = (await (
|
||||
db.WalletObjects.AsNoTracking()
|
||||
.Where(wo => wo.Type == WalletObjectData.Types.Label)
|
||||
.Select(wl => new { wl.WalletId, wl.Id })
|
||||
.ToListAsync()))
|
||||
.Select(o => (o.WalletId, o.Id)).ToHashSet();
|
||||
}
|
||||
|
||||
|
||||
|
||||
next:
|
||||
// var insertedObjectInDBContext
|
||||
// Need to keep track of this hack, or then EF has a bug where he crash on the .Add and get internally
|
||||
// corrupted.
|
||||
var ifuckinghateentityframework = new HashSet<(string WalletId, string Type, string Id)>();
|
||||
using (var db = _dbContextFactory.CreateContext())
|
||||
{
|
||||
Logs.PayServer.LogInformation($"Wallet transaction label importing transactions {startFromOffset}/{total}");
|
||||
var txs = await db.WalletTransactions
|
||||
.OrderByDescending(wt => wt.WalletDataId).ThenBy(wt => wt.TransactionId)
|
||||
.Skip(startFromOffset)
|
||||
.Take(batchCount)
|
||||
.ToArrayAsync();
|
||||
|
||||
foreach (var tx in txs)
|
||||
{
|
||||
// Same as above
|
||||
var ifuckinghateentityframework2 = new HashSet<(string Type, string Id)>();
|
||||
var blob = GetBlobInfo(tx);
|
||||
db.WalletObjects.Add(new Data.WalletObjectData()
|
||||
{
|
||||
WalletId = tx.WalletDataId,
|
||||
Type = Data.WalletObjectData.Types.Tx,
|
||||
Id = tx.TransactionId,
|
||||
Data = string.IsNullOrEmpty(blob.Comment) ? null : new JObject() { ["comment"] = blob.Comment }.ToString()
|
||||
});
|
||||
|
||||
foreach (var label in blob.Labels)
|
||||
{
|
||||
var labelId = label.Key;
|
||||
if (labelId.StartsWith("{", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
try
|
||||
{
|
||||
labelId = JObject.Parse(label.Key)["value"].Value<string>();
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
}
|
||||
if (!existingLabels.Contains((tx.WalletDataId, labelId)))
|
||||
{
|
||||
JObject labelData = new JObject();
|
||||
labelData.Add("color", "#000");
|
||||
db.WalletObjects.Add(new WalletObjectData()
|
||||
{
|
||||
WalletId = tx.WalletDataId,
|
||||
Type = WalletObjectData.Types.Label,
|
||||
Id = labelId,
|
||||
Data = labelData.ToString()
|
||||
});
|
||||
existingLabels.Add((tx.WalletDataId, labelId));
|
||||
}
|
||||
if (ifuckinghateentityframework2.Add((Data.WalletObjectData.Types.Label, labelId)))
|
||||
db.WalletObjectLinks.Add(new WalletObjectLinkData()
|
||||
{
|
||||
WalletId = tx.WalletDataId,
|
||||
ChildType = Data.WalletObjectData.Types.Tx,
|
||||
ChildId = tx.TransactionId,
|
||||
ParentType = Data.WalletObjectData.Types.Label,
|
||||
ParentId = labelId
|
||||
});
|
||||
|
||||
if (label.Value is ReferenceLabel reflabel)
|
||||
{
|
||||
if (IsReferenceLabel(reflabel.Type))
|
||||
{
|
||||
if (ifuckinghateentityframework.Add((tx.WalletDataId, reflabel.Type, reflabel.Reference ?? String.Empty)))
|
||||
db.WalletObjects.Add(new WalletObjectData()
|
||||
{
|
||||
WalletId = tx.WalletDataId,
|
||||
Type = reflabel.Type,
|
||||
Id = reflabel.Reference ?? String.Empty
|
||||
});
|
||||
|
||||
if (ifuckinghateentityframework2.Add((reflabel.Type, reflabel.Reference ?? String.Empty)))
|
||||
db.WalletObjectLinks.Add(new WalletObjectLinkData()
|
||||
{
|
||||
WalletId = tx.WalletDataId,
|
||||
ChildType = Data.WalletObjectData.Types.Tx,
|
||||
ChildId = tx.TransactionId,
|
||||
ParentType = reflabel.Type,
|
||||
ParentId = reflabel.Reference ?? String.Empty
|
||||
});
|
||||
}
|
||||
}
|
||||
else if (label.Value is PayoutLabel payoutLabel)
|
||||
{
|
||||
foreach (var pp in payoutLabel.PullPaymentPayouts)
|
||||
{
|
||||
foreach (var payout in pp.Value)
|
||||
{
|
||||
var payoutData = string.IsNullOrEmpty(pp.Key) ? null : new JObject()
|
||||
{
|
||||
["pullPaymentId"] = pp.Key
|
||||
};
|
||||
if (ifuckinghateentityframework.Add((tx.WalletDataId, "payout", payout)))
|
||||
db.WalletObjects.Add(new WalletObjectData()
|
||||
{
|
||||
WalletId = tx.WalletDataId,
|
||||
Type = "payout",
|
||||
Id = payout,
|
||||
Data = payoutData?.ToString()
|
||||
});
|
||||
if (ifuckinghateentityframework2.Add(("payout", payout)))
|
||||
db.WalletObjectLinks.Add(new WalletObjectLinkData()
|
||||
{
|
||||
WalletId = tx.WalletDataId,
|
||||
ChildType = Data.WalletObjectData.Types.Tx,
|
||||
ChildId = tx.TransactionId,
|
||||
ParentType = "payout",
|
||||
ParentId = payout
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
int retry = 0;
|
||||
retrySave:
|
||||
try
|
||||
{
|
||||
await db.SaveChangesAsync();
|
||||
}
|
||||
catch (DbUpdateException ex) when (retry < 10)
|
||||
{
|
||||
foreach (var entry in ex.Entries)
|
||||
{
|
||||
if (entry.Entity is WalletObjectData wo && (IsReferenceLabel(wo.Type) || wo.Type == "payout"))
|
||||
{
|
||||
await entry.ReloadAsync();
|
||||
}
|
||||
}
|
||||
retry++;
|
||||
goto retrySave;
|
||||
}
|
||||
if (txs.Length < batchCount)
|
||||
{
|
||||
var settings = await _settingsRepository.GetSettingAsync<MigrationSettings>();
|
||||
settings.MigratedTransactionLabels = int.MaxValue;
|
||||
await _settingsRepository.UpdateSetting(settings);
|
||||
Logs.PayServer.LogInformation($"Wallet transaction label successfully migrated");
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
startFromOffset += batchCount;
|
||||
var settings = await _settingsRepository.GetSettingAsync<MigrationSettings>();
|
||||
settings.MigratedTransactionLabels = startFromOffset;
|
||||
await _settingsRepository.UpdateSetting(settings);
|
||||
goto next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsReferenceLabel(string type)
|
||||
{
|
||||
return type == "invoice" ||
|
||||
type == "payment-request" ||
|
||||
type == "app" ||
|
||||
type == "pj-exposed";
|
||||
}
|
||||
#pragma warning restore CS0612 // Type or member is obsolete
|
||||
private async Task MigratedInvoiceTextSearchToDb(int startFromPage)
|
||||
{
|
||||
// deleting legacy DBriize database if present
|
||||
@@ -97,7 +327,7 @@ namespace BTCPayServer.HostedServices
|
||||
textSearch.Add(invoice.RefundMail);
|
||||
// TODO: Are there more things to cache? PaymentData?
|
||||
InvoiceRepository.AddToTextSearch(ctx,
|
||||
new InvoiceData { Id = invoice.Id, InvoiceSearchData = new List<InvoiceSearchData>() },
|
||||
new Data.InvoiceData { Id = invoice.Id, InvoiceSearchData = new List<InvoiceSearchData>() },
|
||||
textSearch.ToArray());
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user