diff --git a/plugins/Makefile b/plugins/Makefile index c9c845519..8259eb64c 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -1,7 +1,3 @@ -PLUGIN_BOOKKEEPER_SRC := plugins/bookkeeper.c -PLUGIN_BOOKKEEPER_HEADER := -PLUGIN_BOOKKEEPER_OBJS := $(PLUGIN_BOOKKEEPER_SRC:.c=.o) - PLUGIN_PAY_SRC := plugins/pay.c PLUGIN_PAY_OBJS := $(PLUGIN_PAY_SRC:.c=.o) @@ -64,7 +60,6 @@ PLUGIN_FUNDER_HEADER := \ PLUGIN_FUNDER_OBJS := $(PLUGIN_FUNDER_SRC:.c=.o) PLUGIN_ALL_SRC := \ - $(PLUGIN_BOOKKEEPER_SRC) \ $(PLUGIN_AUTOCLEAN_SRC) \ $(PLUGIN_chanbackup_SRC) \ $(PLUGIN_BCLI_SRC) \ @@ -81,7 +76,6 @@ PLUGIN_ALL_SRC := \ $(PLUGIN_SPENDER_SRC) PLUGIN_ALL_HEADER := \ - $(PLUGIN_BOOKKEEPER_HEADER) \ $(PLUGIN_LIB_HEADER) \ $(PLUGIN_FUNDER_HEADER) \ $(PLUGIN_PAY_LIB_HEADER) \ @@ -93,7 +87,6 @@ C_PLUGINS := \ plugins/autoclean \ plugins/chanbackup \ plugins/bcli \ - plugins/bookkeeper \ plugins/commando \ plugins/fetchinvoice \ plugins/funder \ @@ -170,11 +163,11 @@ PLUGIN_COMMON_OBJS := \ wire/tlvstream.o \ wire/towire.o +include plugins/bkpr/Makefile + # Make all plugins depend on all plugin headers, for simplicity. $(PLUGIN_ALL_OBJS): $(PLUGIN_ALL_HEADER) -plugins/bookkeeper: bitcoin/chainparams.o common/coin_mvt.o $(PLUGIN_BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) - plugins/pay: bitcoin/chainparams.o $(PLUGIN_PAY_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o common/bolt12.o common/bolt12_merkle.o wire/bolt12$(EXP)_wiregen.o bitcoin/block.o plugins/autoclean: bitcoin/chainparams.o $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) diff --git a/plugins/bkpr/Makefile b/plugins/bkpr/Makefile new file mode 100644 index 000000000..1644feaf4 --- /dev/null +++ b/plugins/bkpr/Makefile @@ -0,0 +1,43 @@ +#! /usr/bin/make + +BOOKKEEPER_PLUGIN_SRC := \ + plugins/bkpr/bookkeeper.c \ + plugins/bkpr/db.c + +BOOKKEEPER_DB_QUERIES := \ + plugins/bkpr/db_sqlite3_sqlgen.c \ + plugins/bkpr/db_postgres_sqlgen.c + +BOOKKEEPER_SRC := $(BOOKKEEPER_PLUGIN_SRC) $(BOOKKEEPER_DB_QUERIES) +BOOKKEEPER_HEADER := plugins/bkpr/db.h +BOOKKEEPER_OBJS := $(BOOKKEEPER_SRC:.c=.o) + +$(BOOKKEEPER_OBJS): $(PLUGIN_LIB_HEADER) $(BOOKKEEPER_HEADER) + +ALL_C_SOURCES += $(BOOKKEEPER_SRC) +ALL_C_HEADERS += $(BOOKKEEPER_HEADER) +ALL_PROGRAMS += plugins/bookkeeper + +plugins/bookkeeper: bitcoin/chainparams.o common/coin_mvt.o $(BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) $(DB_OBJS) + +# The following files contain SQL-annotated statements that we need to extact +BOOKKEEPER_SQL_FILES := \ + $(DB_SQL_FILES) \ + plugins/bkpr/db.c + +plugins/bkpr/statements_gettextgen.po: $(BOOKKEEPER_SQL_FILES) $(FORCE) + @if $(call SHA256STAMP_CHANGED_ALL); then \ + $(call VERBOSE,"xgettext $@",xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(BOOKKEEPER_SQL_FILES) && $(call SHA256STAMP_ALL,# ,)); \ + fi + +plugins/bkpr/db_%_sqlgen.c: plugins/bkpr/statements_gettextgen.po devtools/sql-rewrite.py $(BOOKKEEPER_SQL_FILES) $(FORCE) + @if $(call SHA256STAMP_CHANGED); then \ + $(call VERBOSE,"sql-rewrite $@",devtools/sql-rewrite.py plugins/bkpr/statements_gettextgen.po $* > $@ && $(call SHA256STAMP,//,)); \ + fi + +maintainer-clean: clean +clean: bkpr-maintainer-clean +bkpr-maintainer-clean: + $(RM) plugins/bkpr/statements.po + $(RM) plugins/bkpr/statements_gettextgen.po + $(RM) $(BOOKKEEPER_DB_QUERIES) diff --git a/plugins/bookkeeper.c b/plugins/bkpr/bookkeeper.c similarity index 96% rename from plugins/bookkeeper.c rename to plugins/bkpr/bookkeeper.c index 5326f0e5d..7b04ed61f 100644 --- a/plugins/bookkeeper.c +++ b/plugins/bkpr/bookkeeper.c @@ -2,13 +2,21 @@ #include #include #include +#include #include #include +#include #include #define CHAIN_MOVE "chain_mvt" #define CHANNEL_MOVE "channel_mvt" +/* The database that we store all the accounting data in */ +static struct db *db ; + +// FIXME: make relative to directory we're loaded into +static char *db_dsn = "sqlite3://accounts.sqlite3"; + static struct command_result *json_list_balances(struct command *cmd, const char *buf, const jsmntok_t *params) @@ -321,7 +329,9 @@ static const struct plugin_command commands[] = { static const char *init(struct plugin *p, const char *b, const jsmntok_t *t) { - // FIXME set up database, run migrations + // FIXME: pass in database DSN as an option?? + db = notleak(db_setup(p, p, db_dsn)); + return NULL; } diff --git a/plugins/bkpr/db.c b/plugins/bkpr/db.c new file mode 100644 index 000000000..8fce4f398 --- /dev/null +++ b/plugins/bkpr/db.c @@ -0,0 +1,168 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include + +struct migration { + const char *sql; + void (*func)(struct plugin *p, struct db *db); +}; + +/* Do not reorder or remove elements from this array. + * It is used to migrate existing databases from a prevoius state, based on + * string indicies */ +static struct migration db_migrations[] = { + {SQL("CREATE TABLE version (version INTEGER);"), NULL}, + {SQL("INSERT INTO version VALUES (1);"), NULL}, + {SQL("CREATE TABLE vars (" + " name TEXT" + ", val TEXT" + ", intval INTEGER" + ", blobval BLOB" + ", PRIMARY KEY (name)" + ");"), + NULL}, + {SQL("INSERT INTO vars (" + " name" + ", intval" + ") VALUES (" + " 'data_version'" + ", 0" + ");"), + NULL}, + {SQL("CREATE TABLE accounts (" + " id BIGSERIAL" + ", name TEXT" + ", peer_id BLOB" + ", opened_event_id BIGINT" + ", closed_event_id BIGINT" + ", onchain_resolved_block INTEGER" + ", is_wallet INTEGER" + ", we_opened INTEGER" + ", leased INTEGER" + ", last_updated_ts BIGINT" + ", last_updated_block INTEGER" + ", PRIMARY KEY (id)" + ");"), + NULL}, + {SQL("CREATE TABLE chain_events (" + " id BIGSERIAL" + ", account_id BIGINT REFERENCES accounts(id)" + ", tag TEXT" + ", credit BIGINT" + ", debit BIGINT" + ", output_value BIGINT" + ", currency TEXT" + ", timestamp BIGINT" + ", blockheight INTEGER" + ", utxo_txid BLOB" + ", outnum INTEGER" + ", spending_txid BLOB" + ", PRIMARY KEY (id)" + ");"), + NULL}, + {SQL("CREATE TABLE channel_events (" + " id BIGSERIAL" + ", account_id BIGINT REFERENCES accounts(id)" + ", tag TEXT" + ", credit BIGINT" + ", debit BIGINT" + ", fees BIGINT" + ", currency TEXT" + ", payment_id BLOB" + ", part_id INTEGER" + ", timestamp BIGINT" + ", PRIMARY KEY (id)" + ");"), + NULL}, + {SQL("CREATE TABLE onchain_fees (" + "account_id BIGINT REFERENCES accounts(id)" + ", txid BLOB" + ", amount BIGINT" + ", PRIMARY KEY (account_id, txid)" + ");"), + NULL}, +}; + +static bool db_migrate(struct plugin *p, struct db *db) +{ + /* Read current version from database */ + int current, orig, available; + struct db_stmt *stmt; + + orig = current = db_get_version(db); + available = ARRAY_SIZE(db_migrations) - 1; + + if (current == -1) + plugin_log(p, LOG_INFORM, "Creating database"); + else if (available < current) + plugin_err(p, + "Refusing to migrate down from version %u to %u", + current, available); + else if (current != available) + plugin_log(p, LOG_INFORM, + "Updating database from version %u to %u", + current, available); + + while (current < available) { + current++; + if (db_migrations[current].sql) { + struct db_stmt *stmt = + db_prepare_v2(db, db_migrations[current].sql); + db_exec_prepared_v2(take(stmt)); + } + if (db_migrations[current].func) + db_migrations[current].func(p, db); + } + + /* Finally, update the version number in the version table */ + stmt = db_prepare_v2(db, SQL("UPDATE version SET version=?;")); + db_bind_int(stmt, 0, available); + db_exec_prepared_v2(take(stmt)); + + return current != orig; +} + +/* Implement db_fatal, as a wrapper around fatal. + * We use a ifndef block so that it can get be + * implemented in a test file first, if necessary */ +#ifndef DB_FATAL +#define DB_FATAL +void db_fatal(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + va_end(ap); + + exit(1); +} +#endif /* DB_FATAL */ + +struct db *db_setup(const tal_t *ctx, struct plugin *p, char *db_dsn) +{ + bool migrated; + struct db *db = db_open(ctx, db_dsn); + + db->report_changes_fn = NULL; + + db_begin_transaction(db); + migrated = db_migrate(p, db); + db->data_version = db_data_version_get(db); + db_commit_transaction(db); + + /* This needs to be done outside a transaction, apparently. + * It's a good idea to do this every so often, and on db + * upgrade is a reasonable time. */ + if (migrated && !db->config->vacuum_fn(db)) + db_fatal("Error vacuuming db: %s", db->error); + + return db; +} diff --git a/plugins/bkpr/db.h b/plugins/bkpr/db.h new file mode 100644 index 000000000..0c9663e94 --- /dev/null +++ b/plugins/bkpr/db.h @@ -0,0 +1,11 @@ +#ifndef LIGHTNING_PLUGINS_BKPR_DB_H +#define LIGHTNING_PLUGINS_BKPR_DB_H +#include "config.h" +#include + +struct plugin; +struct db; + +struct db *db_setup(const tal_t *ctx, struct plugin *p, char *db_dsn); + +#endif /* LIGHTNING_PLUGINS_BKPR_DB_H */