bkpr: get rid of crash in listincome

Our consolidate fees had a crash bug (and was pretty convoluted). This
makes it less convoluted and resolves the crash.

The only kinda meh thing is that we have to look up the most recent
timestamp data for the onchain fee entry separately, because of the way
SQL sums work.

plugins/bkpr/incomestmt.c

@@ -246,38 +246,38 @@ static struct income_event *maybe_channel_income(const tal_t *ctx,
 
 static struct onchain_fee **find_consolidated_fees(const tal_t *ctx,
 						   struct db *db,
-						   struct onchain_fee **fees TAKES)
+						   u64 start_time,
+						   u64 end_time)
 {
 	struct fee_sum **sums;
-	struct onchain_fee **updated_fees
+	struct onchain_fee **fee_sums
 		= tal_arr(ctx, struct onchain_fee *, 0);
 
 	sums = calculate_onchain_fee_sums(ctx, db);
 
 	for (size_t i = 0; i < tal_count(sums); i++) {
 		/* Find the last matching feerate's data */
-		for (size_t j = tal_count(fees); j > 0; j--) {
-			struct onchain_fee *fee;
-			if (bitcoin_txid_eq(&fees[j - 1]->txid,
-					    sums[i]->txid)
-			    && fees[j - 1]->acct_db_id == sums[i]->acct_db_id) {
-				fee = tal_steal(updated_fees, fees[j - 1]);
-				fee->credit = sums[i]->fees_paid;
-				fee->debit = AMOUNT_MSAT(0);
-				tal_arr_expand(&updated_fees, fee);
-				fees[j - 1] = NULL;
-				break;
-			}
-		}
+		struct onchain_fee *fee;
 
-		/* It's possible there weren't any fee events
-		 * for this txid in the time period we've selected */
+		if (amount_msat_zero(sums[i]->fees_paid))
+			continue;
+
+		fee = tal(fee_sums, struct onchain_fee);
+		fee->credit = sums[i]->fees_paid;
+		fee->debit = AMOUNT_MSAT(0);
+		fee->currency = tal_steal(fee, sums[i]->currency);
+		fee->acct_name = tal_steal(fee, sums[i]->acct_name);
+		fee->txid = *sums[i]->txid;
+
+		fee->timestamp =
+			onchain_fee_last_timestamp(db, sums[i]->acct_db_id,
+						   sums[i]->txid);
+
+		tal_arr_expand(&fee_sums, fee);
 	}
 
-	if (taken(fees))
-		tal_free(fees);
-
-	return updated_fees;
+	tal_free(sums);
+	return fee_sums;
 }
 
 struct income_event **list_income_events(const tal_t *ctx,
@@ -296,12 +296,15 @@ struct income_event **list_income_events(const tal_t *ctx,
 	channel_events = list_channel_events_timebox(ctx, db,
 						     start_time, end_time);
 	chain_events = list_chain_events_timebox(ctx, db, start_time, end_time);
-	onchain_fees = list_chain_fees_timebox(ctx, db, start_time, end_time);
 	accts = list_accounts(ctx, db);
 
-	if (consolidate_fees)
+	if (consolidate_fees) {
 		onchain_fees = find_consolidated_fees(ctx, db,
-						      take(onchain_fees));
+						      start_time,
+						      end_time);
+	} else
+		onchain_fees = list_chain_fees_timebox(ctx, db,
+						       start_time, end_time);
 
 	evs = tal_arr(ctx, struct income_event *, 0);
 

plugins/bkpr/recorder.c

@@ -214,12 +214,18 @@ struct fee_sum **calculate_onchain_fee_sums(const tal_t *ctx, struct db *db)
 	struct db_stmt *stmt;
 	struct fee_sum **sums;
 	stmt = db_prepare_v2(db, SQL("SELECT"
-				     "  txid"
-				     ", account_id"
-				     ", SUM(credit)"
-				     ", SUM(debit)"
-				     " FROM onchain_fees"
-				     " GROUP BY txid, account_id"
+				     "  of.txid"
+				     ", of.account_id"
+				     ", a.name"
+				     ", of.currency"
+				     ", CAST(SUM(of.credit) AS BIGINT) - CAST(SUM(of.debit) AS BIGINT) as fees"
+				     " FROM onchain_fees of"
+				     " LEFT OUTER JOIN accounts a"
+				     " ON of.account_id = a.id"
+				     " GROUP BY of.txid"
+				     ", of.account_id"
+				     ", a.name"
+				     ", of.currency"
 				     " ORDER BY txid, account_id"));
 
 	db_query_prepared(stmt);
@@ -227,18 +233,16 @@ struct fee_sum **calculate_onchain_fee_sums(const tal_t *ctx, struct db *db)
 	sums = tal_arr(ctx, struct fee_sum *, 0);
 	while (db_step(stmt)) {
 		struct fee_sum *sum;
-		struct amount_msat amt;
-		bool ok;
 
 		sum = tal(sums, struct fee_sum);
 		sum->txid = tal(sum, struct bitcoin_txid);
-		db_col_txid(stmt, "txid", sum->txid);
-		sum->acct_db_id = db_col_u64(stmt, "account_id");
 
-		db_col_amount_msat(stmt, "SUM(credit)", &sum->fees_paid);
-		db_col_amount_msat(stmt, "SUM(debit)", &amt);
-		ok = amount_msat_sub(&sum->fees_paid, sum->fees_paid, amt);
-		assert(ok);
+		db_col_txid(stmt, "of.txid", sum->txid);
+		sum->acct_db_id = db_col_u64(stmt, "of.account_id");
+		sum->acct_name = db_col_strdup(sum, stmt, "a.name");
+		sum->currency = db_col_strdup(sum, stmt, "of.currency");
+		db_col_amount_msat(stmt, "fees", &sum->fees_paid);
+
 		tal_arr_expand(&sums, sum);
 	}
 
@@ -246,6 +250,34 @@ struct fee_sum **calculate_onchain_fee_sums(const tal_t *ctx, struct db *db)
 	return sums;
 }
 
+u64 onchain_fee_last_timestamp(struct db *db,
+			       u64 acct_db_id,
+			       struct bitcoin_txid *txid)
+{
+	struct db_stmt *stmt;
+	u64 timestamp;
+
+	stmt = db_prepare_v2(db, SQL("SELECT"
+				     "  timestamp"
+				     " FROM onchain_fees"
+				     " WHERE account_id = ?"
+				     " AND txid = ?"
+				     " ORDER BY timestamp DESC"));
+
+
+	db_bind_u64(stmt, 0, acct_db_id);
+	db_bind_txid(stmt, 1, txid);
+	db_query_prepared(stmt);
+
+	if (db_step(stmt))
+		timestamp = db_col_u64(stmt, "timestamp");
+	else
+		timestamp = 0;
+
+	tal_free(stmt);
+	return timestamp;
+}
+
 struct fee_sum **find_account_onchain_fees(const tal_t *ctx,
 					   struct db *db,
 					   struct account *acct)

plugins/bkpr/recorder.h

@@ -25,6 +25,8 @@ struct acct_balance {
 
 struct fee_sum {
 	u64 acct_db_id;
+	char *acct_name;
+	char *currency;
 	struct bitcoin_txid *txid;
 	struct amount_msat fees_paid;
 };
@@ -138,6 +140,10 @@ struct fee_sum **find_account_onchain_fees(const tal_t *ctx,
 /* Final all the onchain fees */
 struct fee_sum **calculate_onchain_fee_sums(const tal_t *ctx, struct db *db);
 
+/* Find the last timestamp for the onchain fees for this txid + account */
+u64 onchain_fee_last_timestamp(struct db *db,
+			       u64 acct_db_id,
+			       struct bitcoin_txid *txid);
 /* Add the given account to the database */
 void account_add(struct db *db, struct account *acct);
 /* Given an account name, find that account record */