feerates: consider last three raw values for min/max.

We don't know what our peer is doing, but if we see those values, maybe
they did too, and for longer.  And add the min/max acceptable values
into our JSON API.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

lightningd/chaintopology.c

@@ -290,6 +290,21 @@ static const char *feerate_name(enum feerate feerate)
 /* Mutual recursion via timer. */
 static void next_updatefee_timer(struct chain_topology *topo);
 
+static void init_feerate_history(struct chain_topology *topo,
+				 enum feerate feerate, u32 val)
+{
+	for (size_t i = 0; i < FEE_HISTORY_NUM; i++)
+		topo->feehistory[feerate][i] = val;
+}
+
+static void add_feerate_history(struct chain_topology *topo,
+				enum feerate feerate, u32 val)
+{
+	memmove(&topo->feehistory[feerate][1], &topo->feehistory[feerate][0],
+		(FEE_HISTORY_NUM - 1) * sizeof(u32));
+	topo->feehistory[feerate][0] = val;
+}
+
 /* We sanitize feerates if necessary to put them in descending order. */
 static void update_feerates(struct bitcoind *bitcoind,
 			    const u32 *satoshi_per_kw,
@@ -316,10 +331,13 @@ static void update_feerates(struct bitcoind *bitcoind,
 		/* Initial smoothed feerate is the polled feerate */
 		if (!old_feerates[i]) {
 			old_feerates[i] = feerate;
+			init_feerate_history(topo, i, feerate);
+
 			log_debug(topo->log,
 					  "Smoothed feerate estimate for %s initialized to polled estimate %u",
 					  feerate_name(i), feerate);
-		}
+		} else
+			add_feerate_history(topo, i, feerate);
 
 		/* Smooth the feerate to avoid spikes. */
 		u32 feerate_smooth = feerate * alpha + old_feerates[i] * (1 - alpha);
@@ -398,6 +416,7 @@ static void json_feerates(struct command *cmd,
 	bool missing;
 	const jsmntok_t *style;
 	bool bitcoind_style;
+	u64 mulfactor;
 
 	if (!param(cmd, buffer, params,
 		   p_req("style", json_tok_tok, &style),
@@ -412,14 +431,16 @@ static void json_feerates(struct command *cmd,
 	feerates[FEERATE_NORMAL] = normal ? *normal : 0;
 	feerates[FEERATE_SLOW] = slow ? *slow : 0;
 
-	if (json_tok_streq(buffer, style, "sipa"))
+	if (json_tok_streq(buffer, style, "sipa")) {
 		bitcoind_style = false;
-	else if (json_tok_streq(buffer, style, "bitcoind")) {
+		mulfactor = 1;
+	} else if (json_tok_streq(buffer, style, "bitcoind")) {
 		/* Everyone uses satoshi per kbyte, but we use satoshi per ksipa
 		 * (don't round down to zero though)! */
 		for (size_t i = 0; i < ARRAY_SIZE(feerates); i++)
 			feerates[i] = (feerates[i] + 3) / 4;
 		bitcoind_style = true;
+		mulfactor = 4;
 	} else {
 		command_fail(cmd, JSONRPC2_INVALID_PARAMS, "invalid style");
 		return;
@@ -438,8 +459,6 @@ static void json_feerates(struct command *cmd,
 		feerates[i] = try_get_feerate(topo, i);
 		if (!feerates[i])
 			missing = true;
-		if (bitcoind_style)
-			feerates[i] *= 4;
 	}
 
 	response = new_json_result(cmd);
@@ -448,8 +467,12 @@ static void json_feerates(struct command *cmd,
 	for (size_t i = 0; i < ARRAY_SIZE(feerates); i++) {
 		if (!feerates[i])
 			continue;
-		json_add_num(response, feerate_name(i), feerates[i]);
+		json_add_num(response, feerate_name(i), feerates[i] * mulfactor);
 	}
+	json_add_u64(response, "min_acceptable",
+		     feerate_min(cmd->ld, NULL) * mulfactor);
+	json_add_u64(response, "max_acceptable",
+		     feerate_max(cmd->ld, NULL) * mulfactor);
 	json_object_end(response);
 
 	if (missing)
@@ -717,12 +740,21 @@ u32 feerate_min(struct lightningd *ld, bool *unknown)
 	if (ld->config.ignore_fee_limits)
 		min = 1;
 	else {
-		u32 feerate = try_get_feerate(ld->topology, FEERATE_SLOW);
+		min = try_get_feerate(ld->topology, FEERATE_SLOW);
-		if (!feerate && unknown)
+		if (!min) {
+			if (unknown)
 				*unknown = true;
+		} else {
+			const u32 *hist = ld->topology->feehistory[FEERATE_SLOW];
 
-		/* Set this to half of slow rate (if unknown, will be floor) */
+			/* If one of last three was an outlier, use that. */
-		min = feerate / 2;
+			for (size_t i = 0; i < FEE_HISTORY_NUM; i++) {
+				if (hist[i] < min)
+					min = hist[i];
+			}
+			/* Normally, we use half of slow rate. */
+			min /= 2;
+		}
 	}
 
 	if (min < feerate_floor())
@@ -739,6 +771,7 @@ u32 feerate_min(struct lightningd *ld, bool *unknown)
 u32 feerate_max(struct lightningd *ld, bool *unknown)
 {
 	u32 feerate;
+	const u32 *feehistory = ld->topology->feehistory[FEERATE_URGENT];
 
 	if (unknown)
 		*unknown = false;
@@ -754,6 +787,11 @@ u32 feerate_max(struct lightningd *ld, bool *unknown)
 		return UINT_MAX;
 	}
 
+	/* If one of last three was an outlier, use that. */
+	for (size_t i = 0; i < FEE_HISTORY_NUM; i++) {
+		if (feehistory[i] > feerate)
+			feerate = feehistory[i];
+	}
 	return feerate * ld->config.max_fee_multiplier;
 }
 

lightningd/chaintopology.h

@@ -26,6 +26,9 @@ enum feerate {
 };
 #define NUM_FEERATES (FEERATE_SLOW+1)
 
+/* We keep the last three in case there are outliers (for min/max) */
+#define FEE_HISTORY_NUM 3
+
 /* Off topology->outgoing_txs */
 struct outgoing_tx {
 	struct list_node list;
@@ -83,6 +86,7 @@ struct chain_topology {
 	struct block_map block_map;
 	u32 feerate[NUM_FEERATES];
 	bool feerate_uninitialized;
+	u32 feehistory[NUM_FEERATES][FEE_HISTORY_NUM];
 
 	/* Where to store blockchain info. */
 	struct wallet *wallet;

tests/test_misc.py

@@ -866,36 +866,70 @@ def test_feerates(node_factory):
 
     # Query feerates (shouldn't give any!)
     feerates = l1.rpc.feerates('sipa')
-    assert len(feerates['sipa']) == 0
+    assert len(feerates['sipa']) == 2
     assert feerates['warning'] == 'Some fee estimates unavailable: bitcoind startup?'
     assert 'bitcoind' not in feerates
+    assert feerates['sipa']['max_acceptable'] == 2**32 - 1
+    assert feerates['sipa']['min_acceptable'] == 253
 
     feerates = l1.rpc.feerates('bitcoind')
-    assert len(feerates['bitcoind']) == 0
+    assert len(feerates['bitcoind']) == 2
     assert feerates['warning'] == 'Some fee estimates unavailable: bitcoind startup?'
     assert 'sipa' not in feerates
+    assert feerates['bitcoind']['max_acceptable'] == (2**32 - 1) * 4
+    assert feerates['bitcoind']['min_acceptable'] == 253 * 4
 
     # Now try setting them, one at a time.
     feerates = l1.rpc.feerates('sipa', 15000)
-    assert len(feerates['sipa']) == 1
+    assert len(feerates['sipa']) == 3
     assert feerates['sipa']['urgent'] == 15000
     assert feerates['warning'] == 'Some fee estimates unavailable: bitcoind startup?'
     assert 'bitcoind' not in feerates
+    assert feerates['sipa']['max_acceptable'] == 15000 * 10
+    assert feerates['sipa']['min_acceptable'] == 253
 
     feerates = l1.rpc.feerates('bitcoind', normal=25000)
-    assert len(feerates['bitcoind']) == 2
+    assert len(feerates['bitcoind']) == 4
     assert feerates['bitcoind']['urgent'] == 15000 * 4
     assert feerates['bitcoind']['normal'] == 25000
     assert feerates['warning'] == 'Some fee estimates unavailable: bitcoind startup?'
     assert 'sipa' not in feerates
+    assert feerates['bitcoind']['max_acceptable'] == 15000 * 4 * 10
+    assert feerates['bitcoind']['min_acceptable'] == 253 * 4
 
     feerates = l1.rpc.feerates('sipa', None, None, 5000)
-    assert len(feerates['sipa']) == 3
+    assert len(feerates['sipa']) == 5
     assert feerates['sipa']['urgent'] == 15000
     assert feerates['sipa']['normal'] == 25000 // 4
     assert feerates['sipa']['slow'] == 5000
     assert 'warning' not in feerates
     assert 'bitcoind' not in feerates
+    assert feerates['sipa']['max_acceptable'] == 15000 * 10
+    assert feerates['sipa']['min_acceptable'] == 5000 // 2
+
+    # Now, outliers effect min and max, not so much the smoothed avg.
+    feerates = l1.rpc.feerates('sipa', 30000, None, 600)
+    assert len(feerates['sipa']) == 5
+    assert feerates['sipa']['urgent'] > 15000
+    assert feerates['sipa']['urgent'] < 30000
+    assert feerates['sipa']['normal'] == 25000 // 4
+    assert feerates['sipa']['slow'] < 5000
+    assert feerates['sipa']['slow'] > 600
+    assert 'warning' not in feerates
+    assert 'bitcoind' not in feerates
+    assert feerates['sipa']['max_acceptable'] == 30000 * 10
+    assert feerates['sipa']['min_acceptable'] == 600 // 2
+
+    # Forgotten after 3 more values inserted.
+    feerates = l1.rpc.feerates('sipa', 15000, 25000 // 4, 5000)
+    assert feerates['sipa']['max_acceptable'] == 30000 * 10
+    assert feerates['sipa']['min_acceptable'] == 600 // 2
+    feerates = l1.rpc.feerates('sipa', 15000, 25000 // 4, 5000)
+    assert feerates['sipa']['max_acceptable'] == 30000 * 10
+    assert feerates['sipa']['min_acceptable'] == 600 // 2
+    feerates = l1.rpc.feerates('sipa', 15000, 25000 // 4, 5000)
+    assert feerates['sipa']['max_acceptable'] == 15000 * 10
+    assert feerates['sipa']['min_acceptable'] == 5000 // 2
 
 
 def test_logging(node_factory):