From 5a6489c3aee01baf54472c211f0054ba7ec87e64 Mon Sep 17 00:00:00 2001 From: Michael Schmoock Date: Thu, 23 Jul 2020 20:41:22 +0200 Subject: [PATCH] summary: moves availability code and adds testcases extracts the availability calculations to own testable module --- summary/summary.py | 64 +++++++----------- summary/summary_avail.py | 32 +++++++++ summary/test_summary.py | 139 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 41 deletions(-) create mode 100644 summary/summary_avail.py diff --git a/summary/summary.py b/summary/summary.py index 280d413..f262260 100755 --- a/summary/summary.py +++ b/summary/summary.py @@ -2,7 +2,7 @@ from pyln.client import Plugin, Millisatoshi from packaging import version from collections import namedtuple -from datetime import datetime +from summary_avail import * import pyln.client from math import floor, log10 import requests @@ -30,20 +30,6 @@ summary_description = "Gets summary information about this node.\n"\ "Pass a list of scids to the {exclude} parameter"\ " to exclude some channels from the outputs." -# Global state to measure online% and last_seen -peerstate = {} - - -# ensure an rpc peer is added -def addpeer(p): - pid = p['id'] - if not pid in peerstate: - peerstate[pid] = { - 'connected' : p['connected'], - 'last_seen' : datetime.now() if p['connected'] else None, - 'availability' : 1.0 if p['connected'] else 0.0 - } - class PeerThread(threading.Thread): def __init__(self): @@ -51,36 +37,16 @@ class PeerThread(threading.Thread): self.daemon = True def run(self): - interval = 5 * 60 # collect peer state once in a while - window = 3 * 24 * 60 * 60 # 72hr availability window - count = 0 - # delay initial execution, so peers have a chance to connect on startup - time.sleep(interval) + time.sleep(plugin.avail_interval) while True: - count += 1 - leadwin = max(min(window, count * interval), interval) - samples = leadwin / interval - alpha = 1.0 / samples - beta = 1.0 - alpha - try: - peers = plugin.rpc.listpeers() - for p in peers['peers']: - pid = p['id'] - addpeer(p) - - if p['connected']: - peerstate[pid]['last_seen'] = datetime.now() - peerstate[pid]['connected'] = True - peerstate[pid]['availability'] = 1.0 * alpha + peerstate[pid]['availability'] * beta - else: - peerstate[pid]['connected'] = False - peerstate[pid]['availability'] = 0.0 * alpha + peerstate[pid]['availability'] * beta + rpcpeers = plugin.rpc.listpeers() + trace_availability(plugin, rpcpeers) + time.sleep(plugin.avail_interval) except Exception as ex: plugin.log("[PeerThread] " + str(ex), 'warn') - time.sleep(interval) class PriceThread(threading.Thread): @@ -177,7 +143,7 @@ def summary(plugin, exclude=''): reply['num_gossipers'] = 0 for p in peers['peers']: pid = p['id'] - addpeer(p) + addpeer(plugin, p) active_channel = False for c in p['channels']: if c['state'] != 'CHANNELD_NORMAL': @@ -208,7 +174,7 @@ def summary(plugin, exclude=''): c['private'], p['connected'], c['short_channel_id'], - peerstate[pid]['availability'] + plugin.avail_peerstate[pid]['avail'] )) if not active_channel and p['connected']: @@ -288,6 +254,12 @@ def init(options, configuration, plugin): plugin.currency = options['summary-currency'] plugin.currency_prefix = options['summary-currency-prefix'] plugin.fiat_per_btc = 0 + + plugin.avail_peerstate = {} + plugin.avail_count = 0 + plugin.avail_interval = float(options['summary-availability-interval']) + plugin.avail_window = 60 * 60 * int(options['summary-availability-window']) + info = plugin.rpc.getinfo() # Measure availability @@ -323,4 +295,14 @@ plugin.add_option( 'USD $', 'What prefix to use for currency' ) +plugin.add_option( + 'summary-availability-interval', + 300, + 'How often in seconds the availability should be calculated.' +) +plugin.add_option( + 'summary-availability-window', + 72, + 'How many hours the availability should be averaged over.' +) plugin.run() diff --git a/summary/summary_avail.py b/summary/summary_avail.py new file mode 100644 index 0000000..4d8d219 --- /dev/null +++ b/summary/summary_avail.py @@ -0,0 +1,32 @@ +from datetime import datetime + +# ensure an rpc peer is added +def addpeer(p, rpcpeer): + pid = rpcpeer['id'] + if not pid in p.avail_peerstate: + p.avail_peerstate[pid] = { + 'connected' : rpcpeer['connected'], + 'last_seen' : datetime.now() if rpcpeer['connected'] else None, + 'avail' : 1.0 if rpcpeer['connected'] else 0.0 + } + + +# exponetially smooth online/offline states of peers +def trace_availability(p, rpcpeers): + p.avail_count += 1 + leadwin = max(min(p.avail_window, p.avail_count * p.avail_interval), p.avail_interval) + samples = leadwin / p.avail_interval + alpha = 1.0 / samples + beta = 1.0 - alpha + + for rpcpeer in rpcpeers['peers']: + pid = rpcpeer['id'] + addpeer(p, rpcpeer) + + if rpcpeer['connected']: + p.avail_peerstate[pid]['last_seen'] = datetime.now() + p.avail_peerstate[pid]['connected'] = True + p.avail_peerstate[pid]['avail'] = 1.0 * alpha + p.avail_peerstate[pid]['avail'] * beta + else: + p.avail_peerstate[pid]['connected'] = False + p.avail_peerstate[pid]['avail'] = 0.0 * alpha + p.avail_peerstate[pid]['avail'] * beta diff --git a/summary/test_summary.py b/summary/test_summary.py index 88d4873..7624495 100644 --- a/summary/test_summary.py +++ b/summary/test_summary.py @@ -1,12 +1,151 @@ import subprocess import unittest +from pyln.client import Plugin from pyln.testing.fixtures import * # noqa: F401,F403 from pyln.testing.utils import DEVELOPER +from summary_avail import * + pluginopt = {'plugin': os.path.join(os.path.dirname(__file__), "summary.py")} +# returns a test plugin stub +def get_stub(): + plugin = Plugin() + plugin.avail_peerstate = {} + plugin.avail_count = 0 + plugin.avail_interval = 60 + plugin.avail_window = 3600 + return plugin + + +# tests the 72hr exponential availibility tracing +# tests base algo and peerstate tracing +def test_summary_avail_101(): + # given + plugin = get_stub() + rpcpeers = { + 'peers' : [ + { 'id' : '1', 'connected' : True }, + { 'id' : '2', 'connected' : False }, + { 'id' : '3', 'connected' : True }, + ] + } + + # when + for i in range(100): + trace_availability(plugin, rpcpeers) + + # then + assert(plugin.avail_peerstate['1']['avail'] == 1.0) + assert(plugin.avail_peerstate['2']['avail'] == 0.0) + assert(plugin.avail_peerstate['3']['avail'] == 1.0) + assert(plugin.avail_peerstate['1']['connected'] == True) + assert(plugin.avail_peerstate['2']['connected'] == False) + assert(plugin.avail_peerstate['3']['connected'] == True) + + +# tests for 50% downtime +def test_summary_avail_50(): + # given + plugin = get_stub() + rpcpeers_on = { + 'peers' : [ + { 'id' : '1', 'connected' : True }, + ] + } + rpcpeers_off = { + 'peers' : [ + { 'id' : '1', 'connected' : False }, + ] + } + + # when + for i in range(30): + trace_availability(plugin, rpcpeers_on) + for i in range(30): + trace_availability(plugin, rpcpeers_off) + + # then + assert(round(plugin.avail_peerstate['1']['avail'], 3) == 0.5) + + +# tests for 2/3 downtime +def test_summary_avail_33(): + # given + plugin = get_stub() + rpcpeers_on = { + 'peers' : [ + { 'id' : '1', 'connected' : True }, + ] + } + rpcpeers_off = { + 'peers' : [ + { 'id' : '1', 'connected' : False }, + ] + } + + # when + for i in range(20): + trace_availability(plugin, rpcpeers_on) + for i in range(40): + trace_availability(plugin, rpcpeers_off) + + # then + assert(round(plugin.avail_peerstate['1']['avail'], 3) == 0.333) + + +# tests for 1/3 downtime +def test_summary_avail_66(): + # given + plugin = get_stub() + rpcpeers_on = { + 'peers' : [ + { 'id' : '1', 'connected' : True }, + ] + } + rpcpeers_off = { + 'peers' : [ + { 'id' : '1', 'connected' : False }, + ] + } + + # when + for i in range(40): + trace_availability(plugin, rpcpeers_on) + for i in range(20): + trace_availability(plugin, rpcpeers_off) + + # then + assert(round(plugin.avail_peerstate['1']['avail'], 3) == 0.667) + + +# checks the leading window is smaller if interval count is low +# when a node just started +def test_summary_avail_leadwin(): + # given + plugin = get_stub() + rpcpeers_on = { + 'peers' : [ + { 'id' : '1', 'connected' : True }, + ] + } + rpcpeers_off = { + 'peers' : [ + { 'id' : '1', 'connected' : False }, + ] + } + + # when + trace_availability(plugin, rpcpeers_on) + trace_availability(plugin, rpcpeers_on) + trace_availability(plugin, rpcpeers_off) + + # then + assert(round(plugin.avail_peerstate['1']['avail'], 3) == 0.667) + + def test_summary_start(node_factory): l1 = node_factory.get_node(options=pluginopt) s = l1.rpc.summary()