bkpr: first attempt at database code for accounting

A database scheme and first attempt at drivers for the bookkeeper
database.

Also moves bookkeeper plugin into its own subdirectory
This commit is contained in:
niftynei
2022-07-19 14:37:26 +09:30
committed by Rusty Russell
parent 1a3bfc479f
commit fb951dbbd6
5 changed files with 235 additions and 10 deletions

43
plugins/bkpr/Makefile Normal file
View File

@@ -0,0 +1,43 @@
#! /usr/bin/make
BOOKKEEPER_PLUGIN_SRC := \
plugins/bkpr/bookkeeper.c \
plugins/bkpr/db.c
BOOKKEEPER_DB_QUERIES := \
plugins/bkpr/db_sqlite3_sqlgen.c \
plugins/bkpr/db_postgres_sqlgen.c
BOOKKEEPER_SRC := $(BOOKKEEPER_PLUGIN_SRC) $(BOOKKEEPER_DB_QUERIES)
BOOKKEEPER_HEADER := plugins/bkpr/db.h
BOOKKEEPER_OBJS := $(BOOKKEEPER_SRC:.c=.o)
$(BOOKKEEPER_OBJS): $(PLUGIN_LIB_HEADER) $(BOOKKEEPER_HEADER)
ALL_C_SOURCES += $(BOOKKEEPER_SRC)
ALL_C_HEADERS += $(BOOKKEEPER_HEADER)
ALL_PROGRAMS += plugins/bookkeeper
plugins/bookkeeper: bitcoin/chainparams.o common/coin_mvt.o $(BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) $(DB_OBJS)
# The following files contain SQL-annotated statements that we need to extact
BOOKKEEPER_SQL_FILES := \
$(DB_SQL_FILES) \
plugins/bkpr/db.c
plugins/bkpr/statements_gettextgen.po: $(BOOKKEEPER_SQL_FILES) $(FORCE)
@if $(call SHA256STAMP_CHANGED_ALL); then \
$(call VERBOSE,"xgettext $@",xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(BOOKKEEPER_SQL_FILES) && $(call SHA256STAMP_ALL,# ,)); \
fi
plugins/bkpr/db_%_sqlgen.c: plugins/bkpr/statements_gettextgen.po devtools/sql-rewrite.py $(BOOKKEEPER_SQL_FILES) $(FORCE)
@if $(call SHA256STAMP_CHANGED); then \
$(call VERBOSE,"sql-rewrite $@",devtools/sql-rewrite.py plugins/bkpr/statements_gettextgen.po $* > $@ && $(call SHA256STAMP,//,)); \
fi
maintainer-clean: clean
clean: bkpr-maintainer-clean
bkpr-maintainer-clean:
$(RM) plugins/bkpr/statements.po
$(RM) plugins/bkpr/statements_gettextgen.po
$(RM) $(BOOKKEEPER_DB_QUERIES)

349
plugins/bkpr/bookkeeper.c Normal file
View File

@@ -0,0 +1,349 @@
#include "config.h"
#include <ccan/array_size/array_size.h>
#include <common/coin_mvt.h>
#include <common/json_param.h>
#include <common/memleak.h>
#include <common/node_id.h>
#include <common/type_to_string.h>
#include <plugins/bkpr/db.h>
#include <plugins/libplugin.h>
#define CHAIN_MOVE "chain_mvt"
#define CHANNEL_MOVE "channel_mvt"
/* The database that we store all the accounting data in */
static struct db *db ;
// FIXME: make relative to directory we're loaded into
static char *db_dsn = "sqlite3://accounts.sqlite3";
static struct command_result *json_list_balances(struct command *cmd,
const char *buf,
const jsmntok_t *params)
{
struct json_stream *res;
if (!param(cmd, buf, params, NULL))
return command_param_failed();
res = jsonrpc_stream_success(cmd);
return command_finished(cmd, res);
}
struct account_snap {
char *name;
struct amount_msat amt;
char *coin_type;
};
static struct command_result *json_balance_snapshot(struct command *cmd,
const char *buf,
const jsmntok_t *params)
{
const char *err;
size_t i;
struct node_id node_id;
u32 blockheight;
u64 timestamp;
struct account_snap *snaps;
const jsmntok_t *accounts_tok, *acct_tok,
*snap_tok = json_get_member(buf, params, "balance_snapshot");
if (snap_tok == NULL || snap_tok->type != JSMN_OBJECT)
plugin_err(cmd->plugin,
"`balance_snapshot` payload did not scan %s: %.*s",
"no 'balance_snapshot'", json_tok_full_len(params),
json_tok_full(buf, params));
err = json_scan(cmd, buf, snap_tok,
"{node_id:%"
",blockheight:%"
",timestamp:%}",
JSON_SCAN(json_to_node_id, &node_id),
JSON_SCAN(json_to_number, &blockheight),
JSON_SCAN(json_to_u64, &timestamp));
if (err)
plugin_err(cmd->plugin,
"`balance_snapshot` payload did not scan %s: %.*s",
err, json_tok_full_len(params),
json_tok_full(buf, params));
plugin_log(cmd->plugin, LOG_DBG, "balances for node %s at %d"
" (%"PRIu64")",
type_to_string(tmpctx, struct node_id, &node_id),
blockheight, timestamp);
accounts_tok = json_get_member(buf, snap_tok, "accounts");
if (accounts_tok == NULL || accounts_tok->type != JSMN_ARRAY)
plugin_err(cmd->plugin,
"`balance_snapshot` payload did not scan %s: %.*s",
"no 'balance_snapshot.accounts'",
json_tok_full_len(params),
json_tok_full(buf, params));
snaps = tal_arr(cmd, struct account_snap, accounts_tok->size);
json_for_each_arr(i, acct_tok, accounts_tok) {
struct account_snap s = snaps[i];
err = json_scan(cmd, buf, acct_tok,
"{account_id:%"
",balance_msat:%"
",coin_type:%}",
JSON_SCAN_TAL(tmpctx, json_strdup, &s.name),
JSON_SCAN(json_to_msat, &s.amt),
JSON_SCAN_TAL(tmpctx, json_strdup,
&s.coin_type));
if (err)
plugin_err(cmd->plugin,
"`balance_snapshot` payload did not"
" scan %s: %.*s",
err, json_tok_full_len(params),
json_tok_full(buf, params));
plugin_log(cmd->plugin, LOG_DBG, "account %s has balance %s",
s.name,
type_to_string(tmpctx, struct amount_msat, &s.amt));
}
// FIXME: check balances are ok!
return notification_handled(cmd);
}
static const char *parse_and_log_chain_move(struct command *cmd,
const char *buf,
const jsmntok_t *params,
const struct node_id *node_id,
const char *acct_name STEALS,
const struct amount_msat credit,
const struct amount_msat debit,
const char *coin_type STEALS,
const u32 timestamp,
const enum mvt_tag *tags)
{
struct bitcoin_outpoint outpt;
static struct amount_msat output_value;
struct sha256 *payment_hash = tal(tmpctx, struct sha256);
struct bitcoin_txid *spending_txid = tal(tmpctx, struct bitcoin_txid);
u32 blockheight;
const char *err;
/* Fields we expect on *every* chain movement */
err = json_scan(tmpctx, buf, params,
"{coin_movement:"
"{utxo_txid:%"
",vout:%"
",output_msat:%"
",blockheight:%"
"}}",
JSON_SCAN(json_to_txid, &outpt.txid),
JSON_SCAN(json_to_number, &outpt.n),
JSON_SCAN(json_to_msat, &output_value),
JSON_SCAN(json_to_number, &blockheight));
if (err)
return err;
/* Now try to get out the optional parts */
err = json_scan(tmpctx, buf, params,
"{coin_movement:"
"{txid:%"
"}}",
JSON_SCAN(json_to_txid, spending_txid));
if (err)
spending_txid = tal_free(spending_txid);
/* Now try to get out the optional parts */
err = json_scan(tmpctx, buf, params,
"{coin_movement:"
"{payment_hash:%"
"}}",
JSON_SCAN(json_to_sha256, payment_hash));
if (err)
payment_hash = tal_free(payment_hash);
// FIXME: enter into database
return NULL;
}
static const char *parse_and_log_channel_move(struct command *cmd,
const char *buf,
const jsmntok_t *params,
const struct node_id *node_id,
const char *acct_name STEALS,
const struct amount_msat credit,
const struct amount_msat debit,
const char *coin_type STEALS,
const u32 timestamp,
const enum mvt_tag *tags)
{
struct sha256 payment_hash;
u64 part_id;
struct amount_msat fees;
const char *err;
err = json_scan(tmpctx, buf, params,
"{coin_movement:"
"{payment_hash:%"
",fees:%"
"}}",
JSON_SCAN(json_to_sha256, &payment_hash),
JSON_SCAN(json_to_msat, &fees));
if (err)
return err;
err = json_scan(tmpctx, buf, params,
"{coin_movement:"
"{part_id:%}}",
JSON_SCAN(json_to_u64, &part_id));
if (err)
part_id = 0;
// FIXME: enter into database?
return NULL;
}
static char *parse_tags(const tal_t *ctx,
const char *buf,
const jsmntok_t *tok,
enum mvt_tag **tags)
{
size_t i;
const jsmntok_t *tag_tok,
*tags_tok = json_get_member(buf, tok, "tags");
if (tags_tok == NULL || tags_tok->type != JSMN_ARRAY)
return "Invalid/missing 'tags' field";
*tags = tal_arr(ctx, enum mvt_tag, tags_tok->size);
json_for_each_arr(i, tag_tok, tags_tok) {
if (!json_to_coin_mvt_tag(buf, tag_tok, &(*tags)[i]))
return "Unable to parse 'tags'";
}
return NULL;
}
static struct command_result * json_coin_moved(struct command *cmd,
const char *buf,
const jsmntok_t *params)
{
const char *err, *mvt_type, *acct_name, *coin_type;
struct node_id node_id;
u32 version;
u64 timestamp;
struct amount_msat credit, debit;
enum mvt_tag *tags;
err = json_scan(tmpctx, buf, params,
"{coin_movement:"
"{version:%"
",node_id:%"
",type:%"
",account_id:%"
",credit_msat:%"
",debit_msat:%"
",coin_type:%"
",timestamp:%"
"}}",
JSON_SCAN(json_to_number, &version),
JSON_SCAN(json_to_node_id, &node_id),
JSON_SCAN_TAL(tmpctx, json_strdup, &mvt_type),
JSON_SCAN_TAL(tmpctx, json_strdup, &acct_name),
JSON_SCAN(json_to_msat, &credit),
JSON_SCAN(json_to_msat, &debit),
JSON_SCAN_TAL(tmpctx, json_strdup, &coin_type),
JSON_SCAN(json_to_u64, &timestamp));
if (err)
plugin_err(cmd->plugin,
"`coin_movement` payload did not scan %s: %.*s",
err, json_tok_full_len(params),
json_tok_full(buf, params));
err = parse_tags(cmd, buf,
json_get_member(buf, params, "coin_movement"),
&tags);
if (err)
plugin_err(cmd->plugin,
"`coin_movement` payload did not scan %s: %.*s",
err, json_tok_full_len(params),
json_tok_full(buf, params));
/* We expect version 2 of coin movements */
assert(version == 2);
plugin_log(cmd->plugin, LOG_DBG, "coin_move %d %s -%s %s %"PRIu64,
version,
type_to_string(tmpctx, struct amount_msat, &credit),
type_to_string(tmpctx, struct amount_msat, &debit),
mvt_type, timestamp);
if (streq(mvt_type, CHAIN_MOVE))
err = parse_and_log_chain_move(cmd, buf, params, &node_id,
acct_name, credit, debit,
coin_type, timestamp,
tags);
else {
assert(streq(mvt_type, CHANNEL_MOVE));
err = parse_and_log_channel_move(cmd, buf, params, &node_id,
acct_name, credit, debit,
coin_type, timestamp,
tags);
}
if (err)
plugin_err(cmd->plugin,
"`coin_movement` payload did not scan %s: %.*s",
err, json_tok_full_len(params),
json_tok_full(buf, params));
return notification_handled(cmd);
}
const struct plugin_notification notifs[] = {
{
"coin_movement",
json_coin_moved,
},
{
"balance_snapshot",
json_balance_snapshot,
}
};
static const struct plugin_command commands[] = {
{
"listbalances",
"bookkeeping",
"List current account balances",
"List of current accounts and their balances",
json_list_balances
},
};
static const char *init(struct plugin *p, const char *b, const jsmntok_t *t)
{
// FIXME: pass in database DSN as an option??
db = notleak(db_setup(p, p, db_dsn));
return NULL;
}
int main(int argc, char *argv[])
{
setup_locale();
plugin_main(argv, init, PLUGIN_STATIC, true, NULL,
commands, ARRAY_SIZE(commands),
notifs, ARRAY_SIZE(notifs),
NULL, 0,
NULL, 0,
NULL);
return 0;
}

168
plugins/bkpr/db.c Normal file
View File

@@ -0,0 +1,168 @@
#include "config.h"
#include <ccan/array_size/array_size.h>
#include <db/bindings.h>
#include <db/common.h>
#include <db/exec.h>
#include <db/utils.h>
#include <plugins/bkpr/db.h>
#include <plugins/libplugin.h>
#include <stdio.h>
struct migration {
const char *sql;
void (*func)(struct plugin *p, struct db *db);
};
/* Do not reorder or remove elements from this array.
* It is used to migrate existing databases from a prevoius state, based on
* string indicies */
static struct migration db_migrations[] = {
{SQL("CREATE TABLE version (version INTEGER);"), NULL},
{SQL("INSERT INTO version VALUES (1);"), NULL},
{SQL("CREATE TABLE vars ("
" name TEXT"
", val TEXT"
", intval INTEGER"
", blobval BLOB"
", PRIMARY KEY (name)"
");"),
NULL},
{SQL("INSERT INTO vars ("
" name"
", intval"
") VALUES ("
" 'data_version'"
", 0"
");"),
NULL},
{SQL("CREATE TABLE accounts ("
" id BIGSERIAL"
", name TEXT"
", peer_id BLOB"
", opened_event_id BIGINT"
", closed_event_id BIGINT"
", onchain_resolved_block INTEGER"
", is_wallet INTEGER"
", we_opened INTEGER"
", leased INTEGER"
", last_updated_ts BIGINT"
", last_updated_block INTEGER"
", PRIMARY KEY (id)"
");"),
NULL},
{SQL("CREATE TABLE chain_events ("
" id BIGSERIAL"
", account_id BIGINT REFERENCES accounts(id)"
", tag TEXT"
", credit BIGINT"
", debit BIGINT"
", output_value BIGINT"
", currency TEXT"
", timestamp BIGINT"
", blockheight INTEGER"
", utxo_txid BLOB"
", outnum INTEGER"
", spending_txid BLOB"
", PRIMARY KEY (id)"
");"),
NULL},
{SQL("CREATE TABLE channel_events ("
" id BIGSERIAL"
", account_id BIGINT REFERENCES accounts(id)"
", tag TEXT"
", credit BIGINT"
", debit BIGINT"
", fees BIGINT"
", currency TEXT"
", payment_id BLOB"
", part_id INTEGER"
", timestamp BIGINT"
", PRIMARY KEY (id)"
");"),
NULL},
{SQL("CREATE TABLE onchain_fees ("
"account_id BIGINT REFERENCES accounts(id)"
", txid BLOB"
", amount BIGINT"
", PRIMARY KEY (account_id, txid)"
");"),
NULL},
};
static bool db_migrate(struct plugin *p, struct db *db)
{
/* Read current version from database */
int current, orig, available;
struct db_stmt *stmt;
orig = current = db_get_version(db);
available = ARRAY_SIZE(db_migrations) - 1;
if (current == -1)
plugin_log(p, LOG_INFORM, "Creating database");
else if (available < current)
plugin_err(p,
"Refusing to migrate down from version %u to %u",
current, available);
else if (current != available)
plugin_log(p, LOG_INFORM,
"Updating database from version %u to %u",
current, available);
while (current < available) {
current++;
if (db_migrations[current].sql) {
struct db_stmt *stmt =
db_prepare_v2(db, db_migrations[current].sql);
db_exec_prepared_v2(take(stmt));
}
if (db_migrations[current].func)
db_migrations[current].func(p, db);
}
/* Finally, update the version number in the version table */
stmt = db_prepare_v2(db, SQL("UPDATE version SET version=?;"));
db_bind_int(stmt, 0, available);
db_exec_prepared_v2(take(stmt));
return current != orig;
}
/* Implement db_fatal, as a wrapper around fatal.
* We use a ifndef block so that it can get be
* implemented in a test file first, if necessary */
#ifndef DB_FATAL
#define DB_FATAL
void db_fatal(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
vfprintf(stderr, fmt, ap);
fprintf(stderr, "\n");
va_end(ap);
exit(1);
}
#endif /* DB_FATAL */
struct db *db_setup(const tal_t *ctx, struct plugin *p, char *db_dsn)
{
bool migrated;
struct db *db = db_open(ctx, db_dsn);
db->report_changes_fn = NULL;
db_begin_transaction(db);
migrated = db_migrate(p, db);
db->data_version = db_data_version_get(db);
db_commit_transaction(db);
/* This needs to be done outside a transaction, apparently.
* It's a good idea to do this every so often, and on db
* upgrade is a reasonable time. */
if (migrated && !db->config->vacuum_fn(db))
db_fatal("Error vacuuming db: %s", db->error);
return db;
}

11
plugins/bkpr/db.h Normal file
View File

@@ -0,0 +1,11 @@
#ifndef LIGHTNING_PLUGINS_BKPR_DB_H
#define LIGHTNING_PLUGINS_BKPR_DB_H
#include "config.h"
#include <ccan/tal/tal.h>
struct plugin;
struct db;
struct db *db_setup(const tal_t *ctx, struct plugin *p, char *db_dsn);
#endif /* LIGHTNING_PLUGINS_BKPR_DB_H */