diff --git a/BTCPayServer.Data/Data/WalletTransactionData.cs b/BTCPayServer.Data/Data/WalletTransactionData.cs
index ea6168716..7f7982acb 100644
--- a/BTCPayServer.Data/Data/WalletTransactionData.cs
+++ b/BTCPayServer.Data/Data/WalletTransactionData.cs
@@ -26,11 +26,4 @@ namespace BTCPayServer.Data
                 .WithMany(w => w.WalletTransactions).OnDelete(DeleteBehavior.Cascade);
         }
     }
-
-    public class WalletTransactionInfo
-    {
-        public string Comment { get; set; } = string.Empty;
-        [JsonIgnore]
-        public HashSet<string> Labels { get; set; } = new HashSet<string>();
-    }
 }
diff --git a/BTCPayServer.Tests/UnitTest1.cs b/BTCPayServer.Tests/UnitTest1.cs
index 369317366..297e40a9f 100644
--- a/BTCPayServer.Tests/UnitTest1.cs
+++ b/BTCPayServer.Tests/UnitTest1.cs
@@ -35,6 +35,7 @@ using BTCPayServer.Security.Bitpay;
 using BTCPayServer.Services;
 using BTCPayServer.Services.Apps;
 using BTCPayServer.Services.Invoices;
+using BTCPayServer.Services.Labels;
 using BTCPayServer.Services.Mails;
 using BTCPayServer.Services.Rates;
 using BTCPayServer.Tests.Logging;
@@ -585,6 +586,95 @@ namespace BTCPayServer.Tests
             }
         }
 
+        [Fact]
+        [Trait("Fast", "Fast")]
+        public void CanParseLegacyLabels()
+        {
+            static void AssertContainsRawLabel(WalletTransactionInfo info)
+            {
+                foreach (var item in new[] { "blah", "lol", "hello" })
+                {
+                    Assert.True(info.Labels.ContainsKey(item));
+                    var rawLabel = Assert.IsType<RawLabel>(info.Labels[item]);
+                    Assert.Equal("raw", rawLabel.Type);
+                    Assert.Equal(item, rawLabel.Text);
+                }
+            }
+            var data = new WalletTransactionData();
+            data.Labels = "blah,lol,hello,lol";
+            var info = data.GetBlobInfo();
+            Assert.Equal(3, info.Labels.Count);
+            AssertContainsRawLabel(info);
+            data.SetBlobInfo(info);
+            Assert.Contains("raw", data.Labels);
+            Assert.Contains("{", data.Labels);
+            Assert.Contains("[", data.Labels);
+            info = data.GetBlobInfo();
+            AssertContainsRawLabel(info);
+
+
+            data = new WalletTransactionData()
+            {
+                Labels = "pos",
+                Blob = Encoders.Hex.DecodeData("1f8b08000000000000037abf7b7fb592737e6e6e6a5e89929592522d000000ffff030036bc6ad911000000")
+            };
+            info = data.GetBlobInfo();
+            var label = Assert.Single(info.Labels);
+            Assert.Equal("raw", label.Value.Type);
+            Assert.Equal("pos", label.Value.Text);
+            Assert.Equal("pos", label.Key);
+
+
+            static void AssertContainsLabel(WalletTransactionInfo info)
+            {
+                Assert.Equal(2, info.Labels.Count);
+                var invoiceLabel = Assert.IsType<ReferenceLabel>(info.Labels["invoice"]);
+                Assert.Equal("BFm1MCJPBCDeRoWXvPcwnM", invoiceLabel.Reference);
+                Assert.Equal("invoice", invoiceLabel.Text);
+                Assert.Equal("invoice", invoiceLabel.Type);
+
+                var appLabel = Assert.IsType<ReferenceLabel>(info.Labels["app"]);
+                Assert.Equal("87kj5yKay8mB4UUZcJhZH5TqDKMD3CznjwLjiu1oYZXe", appLabel.Reference);
+                Assert.Equal("app", appLabel.Text);
+                Assert.Equal("app", appLabel.Type);
+            }
+            data = new WalletTransactionData()
+            {
+                Labels = "[\"{\\n  \\\"value\\\": \\\"invoice\\\",\\n  \\\"id\\\": \\\"BFm1MCJPBCDeRoWXvPcwnM\\\"\\n}\",\"{\\n  \\\"value\\\": \\\"app\\\",\\n  \\\"id\\\": \\\"87kj5yKay8mB4UUZcJhZH5TqDKMD3CznjwLjiu1oYZXe\\\"\\n}\"]",
+            };
+            info = data.GetBlobInfo();
+            AssertContainsLabel(info);
+            data.SetBlobInfo(info);
+            info = data.GetBlobInfo();
+            AssertContainsLabel(info);
+
+            static void AssertPayoutLabel(WalletTransactionInfo info)
+            {
+                Assert.Single(info.Labels);
+                var l = Assert.IsType<PayoutLabel>(info.Labels["payout"]);
+                Assert.Equal("pullPaymentId", l.PullPaymentId);
+                Assert.Equal("walletId", l.WalletId);
+                Assert.Equal("payoutId", l.PayoutId);
+            }
+
+            var payoutId = "payoutId";
+            var pullPaymentId = "pullPaymentId";
+            var walletId = "walletId";
+            // How it was serialized before
+
+            data = new WalletTransactionData()
+            {
+                Labels = new JArray(JObject.FromObject(new { value = "payout", id = payoutId, pullPaymentId, walletId })).ToString()
+            };
+            info = data.GetBlobInfo();
+            AssertPayoutLabel(info);
+            data.SetBlobInfo(info);
+            info = data.GetBlobInfo();
+            AssertPayoutLabel(info);
+        }
+
+        
+
         [Fact]
         [Trait("Fast", "Fast")]
         public void DeterministicUTXOSorter()
@@ -1234,8 +1324,8 @@ namespace BTCPayServer.Tests
                 tx = Assert.Single(transactions.Transactions);
 
                 Assert.Equal("hello", tx.Comment);
-                Assert.Contains("test", tx.Labels.Select(l => l.Value));
-                Assert.Contains("test2", tx.Labels.Select(l => l.Value));
+                Assert.Contains("test", tx.Labels.Select(l => l.Text));
+                Assert.Contains("test2", tx.Labels.Select(l => l.Text));
                 Assert.Equal(2, tx.Labels.GroupBy(l => l.Color).Count());
 
                 Assert.IsType<RedirectToActionResult>(
@@ -1246,8 +1336,8 @@ namespace BTCPayServer.Tests
                 tx = Assert.Single(transactions.Transactions);
 
                 Assert.Equal("hello", tx.Comment);
-                Assert.Contains("test", tx.Labels.Select(l => l.Value));
-                Assert.DoesNotContain("test2", tx.Labels.Select(l => l.Value));
+                Assert.Contains("test", tx.Labels.Select(l => l.Text));
+                Assert.DoesNotContain("test2", tx.Labels.Select(l => l.Text));
                 Assert.Single(tx.Labels.GroupBy(l => l.Color));
 
                 var walletInfo = await tester.PayTester.GetService<WalletRepository>().GetWalletInfo(walletId);
diff --git a/BTCPayServer/Controllers/WalletsController.cs b/BTCPayServer/Controllers/WalletsController.cs
index 045af6e72..03d7c0d08 100644
--- a/BTCPayServer/Controllers/WalletsController.cs
+++ b/BTCPayServer/Controllers/WalletsController.cs
@@ -159,12 +159,12 @@ namespace BTCPayServer.Controllers
             if (addlabel != null)
             {
                 addlabel = addlabel.Trim().TrimStart('{').ToLowerInvariant().Replace(',', ' ').Truncate(MaxLabelSize);
-                var labels = _labelFactory.GetLabels(walletBlobInfo, Request);
+                var labels = _labelFactory.GetWalletColoredLabels(walletBlobInfo, Request);
                 if (!walletTransactionsInfo.TryGetValue(transactionId, out var walletTransactionInfo))
                 {
                     walletTransactionInfo = new WalletTransactionInfo();
                 }
-                if (!labels.Any(l => l.Value.Equals(addlabel, StringComparison.OrdinalIgnoreCase)))
+                if (!labels.Any(l => l.Text.Equals(addlabel, StringComparison.OrdinalIgnoreCase)))
                 {
                     List<string> allColors = new List<string>();
                     allColors.AddRange(LabelColorScheme);
@@ -183,7 +183,8 @@ namespace BTCPayServer.Controllers
                     walletBlobInfo.LabelColors.Add(addlabel, chosenColor);
                     await WalletRepository.SetWalletInfo(walletId, walletBlobInfo);
                 }
-                if (walletTransactionInfo.Labels.Add(addlabel))
+                var rawLabel = new RawLabel(addlabel);
+                if (walletTransactionInfo.Labels.TryAdd(rawLabel.Text, rawLabel))
                 {
                     await WalletRepository.SetWalletTransactionInfo(walletId, transactionId, walletTransactionInfo);
                 }
@@ -195,8 +196,8 @@ namespace BTCPayServer.Controllers
                 {
                     if (walletTransactionInfo.Labels.Remove(removelabel))
                     {
-                        var canDelete = !walletTransactionsInfo.SelectMany(txi => txi.Value.Labels).Any(l => l == removelabel);
-                        if (canDelete)
+                        var canDeleteColor = !walletTransactionsInfo.Any(txi => txi.Value.Labels.ContainsKey(removelabel));
+                        if (canDeleteColor)
                         {
                             walletBlobInfo.LabelColors.Remove(removelabel);
                             await WalletRepository.SetWalletInfo(walletId, walletBlobInfo);
@@ -315,14 +316,14 @@ namespace BTCPayServer.Controllers
 
                     if (walletTransactionsInfo.TryGetValue(tx.TransactionId.ToString(), out var transactionInfo))
                     {
-                        var labels = _labelFactory.GetLabels(walletBlob, transactionInfo, Request);
+                        var labels = _labelFactory.ColorizeTransactionLabels(walletBlob, transactionInfo, Request);
                         vm.Labels.AddRange(labels);
                         model.Labels.AddRange(labels);
                         vm.Comment = transactionInfo.Comment;
                     }
 
                     if (labelFilter == null ||
-                        vm.Labels.Any(l => l.Value.Equals(labelFilter, StringComparison.OrdinalIgnoreCase)))
+                        vm.Labels.Any(l => l.Text.Equals(labelFilter, StringComparison.OrdinalIgnoreCase)))
                         model.Transactions.Add(vm);
                 }
 
@@ -547,7 +548,7 @@ namespace BTCPayServer.Controllers
                         Outpoint = coin.OutPoint.ToString(),
                         Amount = coin.Value.GetValue(network),
                         Comment = info?.Comment,
-                        Labels = info == null ? null : _labelFactory.GetLabels(walletBlobAsync, info, Request),
+                        Labels = info == null ? null : _labelFactory.ColorizeTransactionLabels(walletBlobAsync, info, Request),
                         Link = string.Format(CultureInfo.InvariantCulture, network.BlockExplorerLink, coin.OutPoint.Hash.ToString())
                     };
                 }).ToArray();
diff --git a/BTCPayServer/Data/WalletTransactionDataExtensions.cs b/BTCPayServer/Data/WalletTransactionDataExtensions.cs
index 5544e917d..fbdf5d64f 100644
--- a/BTCPayServer/Data/WalletTransactionDataExtensions.cs
+++ b/BTCPayServer/Data/WalletTransactionDataExtensions.cs
@@ -1,32 +1,57 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
+using BTCPayServer.Services.Labels;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
 
 namespace BTCPayServer.Data
 {
+    public class WalletTransactionInfo
+    {
+        public string Comment { get; set; } = string.Empty;
+        [JsonIgnore]
+        public Dictionary<string, Label> Labels { get; set; } = new Dictionary<string, Label>();
+    }
     public static class WalletTransactionDataExtensions
     {
         public static WalletTransactionInfo GetBlobInfo(this WalletTransactionData walletTransactionData)
         {
+            WalletTransactionInfo blobInfo;
             if (walletTransactionData.Blob == null || walletTransactionData.Blob.Length == 0)
-            {
-                return new WalletTransactionInfo();
-            }
-            var blobInfo = JsonConvert.DeserializeObject<WalletTransactionInfo>(ZipUtils.Unzip(walletTransactionData.Blob));
+                blobInfo = new WalletTransactionInfo();
+            else
+                blobInfo = JsonConvert.DeserializeObject<WalletTransactionInfo>(ZipUtils.Unzip(walletTransactionData.Blob));
             if (!string.IsNullOrEmpty(walletTransactionData.Labels))
             {
                 if (walletTransactionData.Labels.StartsWith('['))
                 {
-                    blobInfo.Labels.AddRange(JArray.Parse(walletTransactionData.Labels).Values<string>());
+                    foreach (var jtoken in JArray.Parse(walletTransactionData.Labels))
+                    {
+                        var l = jtoken.Type == JTokenType.String ? Label.Parse(jtoken.Value<string>())
+                                                                : Label.Parse(jtoken.ToString());
+                        blobInfo.Labels.TryAdd(l.Text, l);
+                    }
                 }
                 else
                 {
-                    blobInfo.Labels.AddRange(walletTransactionData.Labels.Split(',',
-                        StringSplitOptions.RemoveEmptyEntries));
+                    // Legacy path
+                    foreach (var token in walletTransactionData.Labels.Split(',',
+                        StringSplitOptions.RemoveEmptyEntries))
+                    {
+                        var l = Label.Parse(token);
+                        blobInfo.Labels.TryAdd(l.Text, l);
+                    }
                 }
             }
             return blobInfo;
         }
+        static JsonSerializerSettings LabelSerializerSettings = new JsonSerializerSettings()
+        { 
+            ContractResolver = new CamelCasePropertyNamesContractResolver(),
+            Formatting = Formatting.None
+        };
         public static void SetBlobInfo(this WalletTransactionData walletTransactionData, WalletTransactionInfo blobInfo)
         {
             if (blobInfo == null)
@@ -35,8 +60,11 @@ namespace BTCPayServer.Data
                 walletTransactionData.Blob = Array.Empty<byte>();
                 return;
             }
-
-            walletTransactionData.Labels = JArray.FromObject(blobInfo.Labels).ToString();
+            walletTransactionData.Labels = new JArray(
+                blobInfo.Labels.Select(l => JsonConvert.SerializeObject(l.Value, LabelSerializerSettings))
+                .Select(l => JObject.Parse(l))
+                .OfType<JToken>()
+                .ToArray()).ToString();
             walletTransactionData.Blob = ZipUtils.Zip(JsonConvert.SerializeObject(blobInfo));
         }
     }
diff --git a/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs b/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs
index ef33d75f1..8b44bcc27 100644
--- a/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs
+++ b/BTCPayServer/HostedServices/TransactionLabelMarkerHostedService.cs
@@ -9,6 +9,7 @@ using BTCPayServer.Payments;
 using BTCPayServer.Payments.Bitcoin;
 using BTCPayServer.Services;
 using BTCPayServer.Services.Apps;
+using BTCPayServer.Services.Labels;
 using BTCPayServer.Services.PaymentRequests;
 using NBitcoin;
 using Newtonsoft.Json.Linq;
@@ -40,7 +41,7 @@ namespace BTCPayServer.HostedServices
             {
                 var walletId = new WalletId(invoiceEvent.Invoice.StoreId, invoiceEvent.Payment.GetCryptoCode());
                 var transactionId = bitcoinLikePaymentData.Outpoint.Hash;
-                var labels = new List<(string color, string label)>
+                var labels = new List<(string color, Label label)>
                 {
                     UpdateTransactionLabel.InvoiceLabelTemplate(invoiceEvent.Invoice.Id)
                 };
@@ -79,14 +80,14 @@ namespace BTCPayServer.HostedServices
 
                     foreach (var label in pair.Value)
                     {
-                        walletBlobInfo.LabelColors.TryAdd(label.label, label.color);
+                        walletBlobInfo.LabelColors.TryAdd(label.label.Text, label.color);
                     }
 
                     await _walletRepository.SetWalletInfo(updateTransactionLabel.WalletId, walletBlobInfo);
                     var update = false;
                     foreach (var label in pair.Value)
                     {
-                        if (walletTransactionInfo.Labels.Add(label.label))
+                        if (walletTransactionInfo.Labels.TryAdd(label.label.Text, label.label))
                         {
                             update = true;
                         }
@@ -108,47 +109,52 @@ namespace BTCPayServer.HostedServices
         {
 
         }
-        public UpdateTransactionLabel(WalletId walletId, uint256 txId, (string color, string label) colorLabel)
+        public UpdateTransactionLabel(WalletId walletId, uint256 txId, (string color, Label label) colorLabel)
         {
             WalletId = walletId;
-            TransactionLabels = new Dictionary<uint256, List<(string color, string label)>>();
-            TransactionLabels.Add(txId, new List<(string color, string label)>() { colorLabel });
+            TransactionLabels = new Dictionary<uint256, List<(string color, Label label)>>();
+            TransactionLabels.Add(txId, new List<(string color, Label label)>() { colorLabel });
         }
-        public UpdateTransactionLabel(WalletId walletId, uint256 txId, List<(string color, string label)> colorLabels)
+        public UpdateTransactionLabel(WalletId walletId, uint256 txId, List<(string color, Label label)> colorLabels)
         {
             WalletId = walletId;
-            TransactionLabels = new Dictionary<uint256, List<(string color, string label)>>();
+            TransactionLabels = new Dictionary<uint256, List<(string color, Label label)>>();
             TransactionLabels.Add(txId, colorLabels);
         }
-        public static (string color, string label) PayjoinLabelTemplate()
+        public static (string color, Label label) PayjoinLabelTemplate()
         {
-            return ("#51b13e", "payjoin");
+            return ("#51b13e", new RawLabel("payjoin"));
         }
 
-        public static (string color, string label) InvoiceLabelTemplate(string invoice)
+        public static (string color, Label label) InvoiceLabelTemplate(string invoice)
         {
-            return ("#cedc21", JObject.FromObject(new { value = "invoice", id = invoice }).ToString());
+            return ("#cedc21", new ReferenceLabel("invoice", invoice));
         }        
-        public static (string color, string label) PaymentRequestLabelTemplate(string paymentRequestId)
+        public static (string color, Label label) PaymentRequestLabelTemplate(string paymentRequestId)
         {
-            return ("#489D77", JObject.FromObject(new { value = "payment-request", id = paymentRequestId }).ToString());
+            return ("#489D77", new ReferenceLabel("payment-request", paymentRequestId));
         }
-        public static (string color, string label) AppLabelTemplate(string appId)
+        public static (string color, Label label) AppLabelTemplate(string appId)
         {
-            return ("#5093B6", JObject.FromObject(new { value = "app", id = appId }).ToString());
+            return ("#5093B6", new ReferenceLabel("app", appId));
         }
 
-        public static (string color, string label) PayjoinExposedLabelTemplate(string invoice)
+        public static (string color, Label label) PayjoinExposedLabelTemplate(string invoice)
         {
-            return ("#51b13e", JObject.FromObject(new { value = "pj-exposed", id = invoice }).ToString());
+            return ("#51b13e", new ReferenceLabel("pj-exposed", invoice));
         }
 
-        public static  (string color, string label) PayoutTemplate(string payoutId, string pullPaymentId, string walletId)
+        public static  (string color, Label label) PayoutTemplate(string payoutId, string pullPaymentId, string walletId)
         {
-            return ("#3F88AF", JObject.FromObject(new { value = "payout", id = payoutId, pullPaymentId, walletId }).ToString());
+            return ("#3F88AF", new PayoutLabel()
+            {
+                PayoutId = payoutId,
+                PullPaymentId = pullPaymentId,
+                WalletId = walletId
+            });
         }
         public WalletId WalletId { get; set; }
-        public Dictionary<uint256, List<(string color, string label)>> TransactionLabels { get; set; }
+        public Dictionary<uint256, List<(string color, Label label)>> TransactionLabels { get; set; }
         public override string ToString()
         {
             var result = new StringBuilder();
diff --git a/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs b/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs
index d63640ddd..087504672 100644
--- a/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs
+++ b/BTCPayServer/Models/WalletViewModels/ListTransactionsViewModel.cs
@@ -15,9 +15,9 @@ namespace BTCPayServer.Models.WalletViewModels
             public string Link { get; set; }
             public bool Positive { get; set; }
             public string Balance { get; set; }
-            public HashSet<Label> Labels { get; set; } = new HashSet<Label>();
+            public HashSet<ColoredLabel> Labels { get; set; } = new HashSet<ColoredLabel>();
         }
-        public HashSet<Label> Labels { get; set; } = new HashSet<Label>();
+        public HashSet<ColoredLabel> Labels { get; set; } = new HashSet<ColoredLabel>();
         public List<TransactionViewModel> Transactions { get; set; } = new List<TransactionViewModel>();
         public int Skip { get; set; }
         public int Count { get; set; }
diff --git a/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs b/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs
index f492a2f45..fc583553e 100644
--- a/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs
+++ b/BTCPayServer/Models/WalletViewModels/WalletSendModel.cs
@@ -68,7 +68,7 @@ namespace BTCPayServer.Models.WalletViewModels
 
         public class InputSelectionOption
         {
-            public IEnumerable<Label> Labels { get; set; }
+            public IEnumerable<ColoredLabel> Labels { get; set; }
             public string Comment { get; set; }
             public decimal Amount { get; set; }
             public string Outpoint { get; set; }
diff --git a/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs b/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs
index f35d90d45..4bf91258a 100644
--- a/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs
+++ b/BTCPayServer/Payments/PayJoin/PayJoinEndpointController.cs
@@ -12,6 +12,7 @@ using BTCPayServer.HostedServices;
 using BTCPayServer.Payments.Bitcoin;
 using BTCPayServer.Services;
 using BTCPayServer.Services.Invoices;
+using BTCPayServer.Services.Labels;
 using BTCPayServer.Services.Stores;
 using BTCPayServer.Services.Wallets;
 using Microsoft.AspNetCore.Cors;
@@ -462,8 +463,8 @@ namespace BTCPayServer.Payments.PayJoin
             {
                 WalletId = new WalletId(invoice.StoreId, network.CryptoCode),
                 TransactionLabels = selectedUTXOs.GroupBy(pair => pair.Key.Hash).Select(utxo =>
-                       new KeyValuePair<uint256, List<(string color, string label)>>(utxo.Key,
-                           new List<(string color, string label)>()
+                       new KeyValuePair<uint256, List<(string color, Label label)>>(utxo.Key,
+                           new List<(string color, Label label)>()
                            {
                                 UpdateTransactionLabel.PayjoinExposedLabelTemplate(invoice.Id)
                            }))
diff --git a/BTCPayServer/Services/Labels/ColoredLabel.cs b/BTCPayServer/Services/Labels/ColoredLabel.cs
new file mode 100644
index 000000000..9ff06c498
--- /dev/null
+++ b/BTCPayServer/Services/Labels/ColoredLabel.cs
@@ -0,0 +1,43 @@
+using System;
+
+namespace BTCPayServer.Services.Labels
+{
+    public class ColoredLabel
+    {
+        internal ColoredLabel()
+        {
+        }
+
+        public string Text { get; internal set; }
+        public string Color { get; internal set; }
+        public string Link { get; internal set; }
+        public string Tooltip { get; internal set; }
+
+        public override bool Equals(object obj)
+        {
+            ColoredLabel item = obj as ColoredLabel;
+            if (item == null)
+                return false;
+            return Text.Equals(item.Text, StringComparison.OrdinalIgnoreCase);
+        }
+
+        public static bool operator ==(ColoredLabel a, ColoredLabel b)
+        {
+            if (System.Object.ReferenceEquals(a, b))
+                return true;
+            if (((object)a == null) || ((object)b == null))
+                return false;
+            return a.Text == b.Text;
+        }
+
+        public static bool operator !=(ColoredLabel a, ColoredLabel b)
+        {
+            return !(a == b);
+        }
+
+        public override int GetHashCode()
+        {
+            return Text.GetHashCode(StringComparison.OrdinalIgnoreCase);
+        }
+    }
+}
diff --git a/BTCPayServer/Services/Labels/Label.cs b/BTCPayServer/Services/Labels/Label.cs
index f69d7cc62..c686d86a3 100644
--- a/BTCPayServer/Services/Labels/Label.cs
+++ b/BTCPayServer/Services/Labels/Label.cs
@@ -1,44 +1,124 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace BTCPayServer.Services.Labels
 {
-    public class Label
+    public abstract class Label
     {
-        internal Label()
+        public string Type { get; set; }
+        public string Text { get; set; }
+        static void FixLegacy(JObject jObj, ReferenceLabel refLabel)
         {
+            if (refLabel.Reference is null)
+                refLabel.Reference = jObj["id"].Value<string>();
+            FixLegacy(jObj, (Label)refLabel);
         }
-
-        public string Value { get; internal set; }
-        public string RawValue { get; internal set; }
-        public string Color { get; internal set; }
-        public string Link { get; internal set; }
-        public string Tooltip { get; internal set; }
-
-        public override bool Equals(object obj)
+        static void FixLegacy(JObject jObj, PayoutLabel payoutLabel)
         {
-            Label item = obj as Label;
-            if (item == null)
-                return false;
-            return Value.Equals(item.Value, StringComparison.OrdinalIgnoreCase);
+            if (payoutLabel.PayoutId is null)
+                payoutLabel.PayoutId = jObj["id"].Value<string>();
+            FixLegacy(jObj, (Label)payoutLabel);
         }
-
-        public static bool operator ==(Label a, Label b)
+        static void FixLegacy(JObject jObj, Label label)
         {
-            if (System.Object.ReferenceEquals(a, b))
-                return true;
-            if (((object)a == null) || ((object)b == null))
-                return false;
-            return a.Value == b.Value;
+            if (label.Type is null)
+                label.Type = jObj["value"].Value<string>();
+            if (label.Text is null)
+                label.Text = label.Type;
         }
-
-        public static bool operator !=(Label a, Label b)
+        static void FixLegacy(JObject jObj, RawLabel rawLabel)
         {
-            return !(a == b);
+            rawLabel.Type = "raw";
+            FixLegacy(jObj, (Label)rawLabel);
         }
-
-        public override int GetHashCode()
+        public static Label Parse(string str)
         {
-            return Value.GetHashCode(StringComparison.OrdinalIgnoreCase);
+            if (str == null)
+                throw new ArgumentNullException(nameof(str));
+            if (str.StartsWith("{", StringComparison.InvariantCultureIgnoreCase))
+            {
+                var jObj = JObject.Parse(str);
+                string type = null;
+                // Legacy label
+                if (!jObj.ContainsKey("type"))
+                {
+                    type = jObj["value"].Value<string>();
+                }
+                else
+                {
+                    type = jObj["type"].Value<string>();
+                }
+
+                switch (type)
+                {
+                    case "raw":
+                        var rawLabel = JsonConvert.DeserializeObject<RawLabel>(str);
+                        FixLegacy(jObj, rawLabel);
+                        return rawLabel;
+                    case "invoice":
+                    case "payment-request":
+                    case "app":
+                    case "pj-exposed":
+                        var refLabel = JsonConvert.DeserializeObject<ReferenceLabel>(str);
+                        FixLegacy(jObj, refLabel);
+                        return refLabel;
+                    case "payout":
+                        var payoutLabel = JsonConvert.DeserializeObject<PayoutLabel>(str);
+                        FixLegacy(jObj, payoutLabel);
+                        return payoutLabel;
+                    default:
+                        // Legacy
+                        return new RawLabel(jObj["value"].Value<string>());
+                }
+            }
+            else
+            {
+                return new RawLabel(str);
+            }
+        }
+        [JsonExtensionData]
+        public IDictionary<string, JToken> AdditionalData { get; set; }
+    }
+
+    public class RawLabel : Label
+    {
+        public RawLabel()
+        {
+            Type = "raw";
+        }
+        public RawLabel(string text) : this()
+        {
+            Text = text;
         }
     }
+    public class ReferenceLabel : Label
+    {
+        public ReferenceLabel()
+        {
+
+        }
+        public ReferenceLabel(string type, string reference)
+        {
+            Text = type;
+            Reference = reference;
+            Type = type;
+        }
+        [JsonProperty("ref")]
+        public string Reference { get; set; }
+    }
+    public class PayoutLabel : Label
+    {
+        public PayoutLabel()
+        {
+            Type = "payout";
+            Text = "payout";
+        }
+        public string PayoutId { get; set; }
+        public string WalletId { get; set; }
+        public string PullPaymentId { get; set; }
+    }
 }
diff --git a/BTCPayServer/Services/Labels/LabelFactory.cs b/BTCPayServer/Services/Labels/LabelFactory.cs
index 874184cf5..dcf21bf44 100644
--- a/BTCPayServer/Services/Labels/LabelFactory.cs
+++ b/BTCPayServer/Services/Labels/LabelFactory.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using Amazon.Util.Internal.PlatformServices;
 using BTCPayServer.Data;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
@@ -17,104 +18,79 @@ namespace BTCPayServer.Services.Labels
             _linkGenerator = linkGenerator;
         }
 
-        public IEnumerable<Label> GetLabels(WalletBlobInfo walletBlobInfo, WalletTransactionInfo transactionInfo,
+        public IEnumerable<ColoredLabel> ColorizeTransactionLabels(WalletBlobInfo walletBlobInfo, WalletTransactionInfo transactionInfo,
             HttpRequest request)
         {
             foreach (var label in transactionInfo.Labels)
             {
-                if (walletBlobInfo.LabelColors.TryGetValue(label, out var color))
+                if (walletBlobInfo.LabelColors.TryGetValue(label.Value.Text, out var color))
                 {
-                    yield return CreateLabel(label, color, request);
+                    yield return CreateLabel(label.Value, color, request);
                 }
             }
         }
 
-        public IEnumerable<Label> GetLabels(WalletBlobInfo walletBlobInfo, HttpRequest request)
+        public IEnumerable<ColoredLabel> GetWalletColoredLabels(WalletBlobInfo walletBlobInfo, HttpRequest request)
         {
             foreach (var kv in walletBlobInfo.LabelColors)
             {
-                yield return CreateLabel(kv.Key, kv.Value, request);
+                yield return CreateLabel(new RawLabel() { Text = kv.Key }, kv.Value, request);
             }
         }
 
-        private Label CreateLabel(string value, string color, HttpRequest request)
+        private ColoredLabel CreateLabel(Label uncoloredLabel, string color, HttpRequest request)
         {
-            if (value == null)
-                throw new ArgumentNullException(nameof(value));
+            if (uncoloredLabel == null)
+                throw new ArgumentNullException(nameof(uncoloredLabel));
             if (color == null)
                 throw new ArgumentNullException(nameof(color));
-            if (value.StartsWith("{", StringComparison.InvariantCultureIgnoreCase))
+
+            ColoredLabel coloredLabel = new ColoredLabel()
             {
-                var jObj = JObject.Parse(value);
-                if (jObj.ContainsKey("value"))
+                Text = uncoloredLabel.Text,
+                Color = color
+            };
+            if (uncoloredLabel is ReferenceLabel refLabel)
+            {
+                var refInLabel = string.IsNullOrEmpty(refLabel.Reference) ? string.Empty : $"({refLabel.Reference})";
+                switch (uncoloredLabel.Type)
                 {
-                    var id = jObj.ContainsKey("id") ? jObj["id"].Value<string>() : string.Empty;
-                    var idInLabel = string.IsNullOrEmpty(id) ? string.Empty : $"({id})";
-                    
-                    switch (jObj["value"].Value<string>())
-                    {
-                        case "invoice":
-                            return new Label()
-                            {
-                                RawValue = value,
-                                Value = "invoice",
-                                Color = color,
-                                Tooltip = $"Received through an invoice {idInLabel}",
-                                Link = string.IsNullOrEmpty(id)
-                                    ? null
-                                    : _linkGenerator.InvoiceLink(id, request.Scheme, request.Host, request.PathBase)
-                            };
-                        case "payment-request":
-                            return new Label()
-                            {
-                                RawValue = value,
-                                Value = "payment-request",
-                                Color = color,
-                                Tooltip = $"Received through a payment request {idInLabel}",
-                                Link = string.IsNullOrEmpty(id)
-                                    ? null
-                                    : _linkGenerator.PaymentRequestLink(id, request.Scheme, request.Host, request.PathBase)
-                            };
-                        case "app":
-                            return new Label()
-                            {
-                                RawValue = value,
-                                Value = "app",
-                                Color = color,
-                                Tooltip = $"Received through an app {idInLabel}",
-                                Link = string.IsNullOrEmpty(id)
-                                    ? null
-                                    : _linkGenerator.AppLink(id, request.Scheme, request.Host, request.PathBase)
-                            };
-                        case "pj-exposed":
-                            return new Label()
-                            {
-                                RawValue = value,
-                                Value = "payjoin-exposed",
-                                Color = color,
-                                Tooltip = $"This utxo was exposed through a payjoin proposal for an invoice {idInLabel}",
-                                Link = string.IsNullOrEmpty(id)
-                                    ? null
-                                    : _linkGenerator.InvoiceLink(id, request.Scheme, request.Host, request.PathBase)
-                            };
-                        case "payout":
-                            return new Label()
-                            {
-                                RawValue = value,
-                                Value = "payout",
-                                Color = color,
-                                Tooltip = $"Paid a payout of a pull payment ({jObj["pullPaymentId"].Value<string>()})",
-                                Link = string.IsNullOrEmpty(id)
-                                    ? null
-                                    : _linkGenerator.PayoutLink(jObj["walletId"].Value<string>(),
-                                        jObj["pullPaymentId"].Value<string>(), request.Scheme, request.Host,
-                                        request.PathBase)
-                            };
-                    }
+                    case "invoice":
+                        coloredLabel.Tooltip = $"Received through an invoice {refInLabel}";
+                        coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference)
+                                ? null
+                                : _linkGenerator.InvoiceLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase);
+                        break;
+                    case "payment-request":
+                        coloredLabel.Tooltip = $"Received through a payment request {refInLabel}";
+                        coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference)
+                                ? null
+                                : _linkGenerator.PaymentRequestLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase);
+                        break;
+                    case "app":
+                        coloredLabel.Tooltip = $"Received through an app {refInLabel}";
+                        coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference)
+                            ? null
+                            : _linkGenerator.AppLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase);
+                        break;
+                    case "pj-exposed":
+                        coloredLabel.Tooltip = $"This utxo was exposed through a payjoin proposal for an invoice {refInLabel}";
+                        coloredLabel.Link = string.IsNullOrEmpty(refLabel.Reference)
+                            ? null
+                            : _linkGenerator.InvoiceLink(refLabel.Reference, request.Scheme, request.Host, request.PathBase);
+                        break;
                 }
             }
-
-            return new Label() { RawValue = value, Value = value, Color = color };
+            else if (uncoloredLabel is PayoutLabel payoutLabel)
+            {
+                coloredLabel.Tooltip = $"Paid a payout of a pull payment ({payoutLabel.PullPaymentId})";
+                coloredLabel.Link = string.IsNullOrEmpty(payoutLabel.PullPaymentId) || string.IsNullOrEmpty(payoutLabel.WalletId)
+                    ? null
+                    : _linkGenerator.PayoutLink(payoutLabel.WalletId,
+                        payoutLabel.PullPaymentId, request.Scheme, request.Host,
+                        request.PathBase);
+            }
+            return coloredLabel;
         }
     }
 }
diff --git a/BTCPayServer/Views/Wallets/CoinSelection.cshtml b/BTCPayServer/Views/Wallets/CoinSelection.cshtml
index 3f50ca28a..314396685 100644
--- a/BTCPayServer/Views/Wallets/CoinSelection.cshtml
+++ b/BTCPayServer/Views/Wallets/CoinSelection.cshtml
@@ -48,14 +48,14 @@
                         class="badge badge-primary badge-pill ml-2"
                         v-for="label of item.labels"
                         v-bind:style="{ 'background-color': label.color}"
-                        key="label.value">
+                        key="label.text">
 
                         <span v-if="!label.link" data-toggle="tooltip" v-tooltip="label.tooltip">
-                            {{label.value}}
+                            {{label.text}}
                         </span>
 
-                        <a :href="label.link" target="_blank"v-if="label.link" data-toggle="tooltip" v-tooltip="label.tooltip">
-                            {{label.value}}
+                        <a :href="label.link" target="_blank" v-if="label.link" data-toggle="tooltip" v-tooltip="label.tooltip">
+                            {{label.text}}
                             <i class="fa fa-info-circle"></i>
                         </a>
                     </span>
diff --git a/BTCPayServer/Views/Wallets/WalletTransactions.cshtml b/BTCPayServer/Views/Wallets/WalletTransactions.cshtml
index d1c155c56..45ce144b9 100644
--- a/BTCPayServer/Views/Wallets/WalletTransactions.cshtml
+++ b/BTCPayServer/Views/Wallets/WalletTransactions.cshtml
@@ -81,8 +81,8 @@
                 <span class="mr-2">Filter by label:</span>
                 @foreach (var label in Model.Labels)
                 {
-                    <a asp-route-labelFilter="@label.Value" class="badge mr-2 my-1 position-relative text-white d-block"
-                       style="background-color: @label.Color;"><span class="text-white">@label.Value</span></a>
+                    <a asp-route-labelFilter="@label.Text" class="badge mr-2 my-1 position-relative text-white d-block"
+                       style="background-color: @label.Color;"><span class="text-white">@label.Text</span></a>
                 }
             </div>
         </div>
@@ -123,7 +123,7 @@
                                         style="background-color: @label.Color; padding-right: 16px; z-index: 1;"
                                         data-toggle="tooltip"
                                         title="@label.Tooltip">
-                                        <a asp-route-labelFilter="@label.Value" class="text-white">@label.Value</a>
+                                        <a asp-route-labelFilter="@label.Text" class="text-white">@label.Text</a>
 
                                         <form
                                             asp-route-walletId="@this.Context.GetRouteValue("walletId")"
@@ -135,7 +135,7 @@
                                                 name="removelabel"
                                                 style="color: @label.Color; filter: brightness(0.5);"
                                                 type="submit"
-                                                value="@label.RawValue">
+                                                value="@label.Text">
                                                 <span class="fa fa-close"></span>
                                             </button>
                                         </form>
@@ -184,11 +184,11 @@
                                             {
                                                 @if (transaction.Labels.Contains(label))
                                                 {
-                                                    <button name="removelabel" class="btn btn-sm" type="submit" value="@label.Value"><span class="badge" style="display:block;background-color:@label.Color;color:white;text-align:left;"><span class="fa fa-check"></span> @label.Value</span></button>
+                                                    <button name="removelabel" class="btn btn-sm" type="submit" value="@label.Text"><span class="badge" style="display:block;background-color:@label.Color;color:white;text-align:left;"><span class="fa fa-check"></span> @label.Text</span></button>
                                                 }
                                                 else
                                                 {
-                                                    <button name="addlabelclick" class="btn btn-sm" type="submit" value="@label.Value"><span class="badge" style="display:block;background-color:@label.Color;color:white;text-align:left;">@label.Value</span></button>
+                                                    <button name="addlabelclick" class="btn btn-sm" type="submit" value="@label.Text"><span class="badge" style="display:block;background-color:@label.Color;color:white;text-align:left;">@label.Text</span></button>
                                                 }
                                             }
                                         }