mirror of
https://github.com/aljazceru/lightning.git
synced 2025-12-20 23:54:22 +01:00
bkpr: first attempt at database code for accounting
A database scheme and first attempt at drivers for the bookkeeper database. Also moves bookkeeper plugin into its own subdirectory
This commit is contained in:
niftynei
@@ -1,7 +1,3 @@
|
||||
PLUGIN_BOOKKEEPER_SRC := plugins/bookkeeper.c
|
||||
PLUGIN_BOOKKEEPER_HEADER :=
|
||||
PLUGIN_BOOKKEEPER_OBJS := $(PLUGIN_BOOKKEEPER_SRC:.c=.o)
|
||||
|
||||
PLUGIN_PAY_SRC := plugins/pay.c
|
||||
PLUGIN_PAY_OBJS := $(PLUGIN_PAY_SRC:.c=.o)
|
||||
|
||||
@@ -64,7 +60,6 @@ PLUGIN_FUNDER_HEADER := \
|
||||
PLUGIN_FUNDER_OBJS := $(PLUGIN_FUNDER_SRC:.c=.o)
|
||||
|
||||
PLUGIN_ALL_SRC := \
|
||||
$(PLUGIN_BOOKKEEPER_SRC) \
|
||||
$(PLUGIN_AUTOCLEAN_SRC) \
|
||||
$(PLUGIN_chanbackup_SRC) \
|
||||
$(PLUGIN_BCLI_SRC) \
|
||||
@@ -81,7 +76,6 @@ PLUGIN_ALL_SRC := \
|
||||
$(PLUGIN_SPENDER_SRC)
|
||||
|
||||
PLUGIN_ALL_HEADER := \
|
||||
$(PLUGIN_BOOKKEEPER_HEADER) \
|
||||
$(PLUGIN_LIB_HEADER) \
|
||||
$(PLUGIN_FUNDER_HEADER) \
|
||||
$(PLUGIN_PAY_LIB_HEADER) \
|
||||
@@ -93,7 +87,6 @@ C_PLUGINS := \
|
||||
plugins/autoclean \
|
||||
plugins/chanbackup \
|
||||
plugins/bcli \
|
||||
plugins/bookkeeper \
|
||||
plugins/commando \
|
||||
plugins/fetchinvoice \
|
||||
plugins/funder \
|
||||
@@ -170,11 +163,11 @@ PLUGIN_COMMON_OBJS := \
|
||||
wire/tlvstream.o \
|
||||
wire/towire.o
|
||||
|
||||
include plugins/bkpr/Makefile
|
||||
|
||||
# Make all plugins depend on all plugin headers, for simplicity.
|
||||
$(PLUGIN_ALL_OBJS): $(PLUGIN_ALL_HEADER)
|
||||
|
||||
plugins/bookkeeper: bitcoin/chainparams.o common/coin_mvt.o $(PLUGIN_BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS)
|
||||
|
||||
plugins/pay: bitcoin/chainparams.o $(PLUGIN_PAY_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o common/bolt12.o common/bolt12_merkle.o wire/bolt12$(EXP)_wiregen.o bitcoin/block.o
|
||||
|
||||
plugins/autoclean: bitcoin/chainparams.o $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS)
|
||||
|
||||
43
plugins/bkpr/Makefile
Normal file
43
plugins/bkpr/Makefile
Normal file
@@ -0,0 +1,43 @@
|
||||
#! /usr/bin/make
|
||||
|
||||
BOOKKEEPER_PLUGIN_SRC := \
|
||||
plugins/bkpr/bookkeeper.c \
|
||||
plugins/bkpr/db.c
|
||||
|
||||
BOOKKEEPER_DB_QUERIES := \
|
||||
plugins/bkpr/db_sqlite3_sqlgen.c \
|
||||
plugins/bkpr/db_postgres_sqlgen.c
|
||||
|
||||
BOOKKEEPER_SRC := $(BOOKKEEPER_PLUGIN_SRC) $(BOOKKEEPER_DB_QUERIES)
|
||||
BOOKKEEPER_HEADER := plugins/bkpr/db.h
|
||||
BOOKKEEPER_OBJS := $(BOOKKEEPER_SRC:.c=.o)
|
||||
|
||||
$(BOOKKEEPER_OBJS): $(PLUGIN_LIB_HEADER) $(BOOKKEEPER_HEADER)
|
||||
|
||||
ALL_C_SOURCES += $(BOOKKEEPER_SRC)
|
||||
ALL_C_HEADERS += $(BOOKKEEPER_HEADER)
|
||||
ALL_PROGRAMS += plugins/bookkeeper
|
||||
|
||||
plugins/bookkeeper: bitcoin/chainparams.o common/coin_mvt.o $(BOOKKEEPER_OBJS) $(PLUGIN_LIB_OBJS) $(JSMN_OBJTS) $(PLUGIN_COMMON_OBJS) $(WIRE_OBJS) $(DB_OBJS)
|
||||
|
||||
# The following files contain SQL-annotated statements that we need to extact
|
||||
BOOKKEEPER_SQL_FILES := \
|
||||
$(DB_SQL_FILES) \
|
||||
plugins/bkpr/db.c
|
||||
|
||||
plugins/bkpr/statements_gettextgen.po: $(BOOKKEEPER_SQL_FILES) $(FORCE)
|
||||
@if $(call SHA256STAMP_CHANGED_ALL); then \
|
||||
$(call VERBOSE,"xgettext $@",xgettext -kNAMED_SQL -kSQL --add-location --no-wrap --omit-header -o $@ $(BOOKKEEPER_SQL_FILES) && $(call SHA256STAMP_ALL,# ,)); \
|
||||
fi
|
||||
|
||||
plugins/bkpr/db_%_sqlgen.c: plugins/bkpr/statements_gettextgen.po devtools/sql-rewrite.py $(BOOKKEEPER_SQL_FILES) $(FORCE)
|
||||
@if $(call SHA256STAMP_CHANGED); then \
|
||||
$(call VERBOSE,"sql-rewrite $@",devtools/sql-rewrite.py plugins/bkpr/statements_gettextgen.po $* > $@ && $(call SHA256STAMP,//,)); \
|
||||
fi
|
||||
|
||||
maintainer-clean: clean
|
||||
clean: bkpr-maintainer-clean
|
||||
bkpr-maintainer-clean:
|
||||
$(RM) plugins/bkpr/statements.po
|
||||
$(RM) plugins/bkpr/statements_gettextgen.po
|
||||
$(RM) $(BOOKKEEPER_DB_QUERIES)
|
||||
@@ -2,13 +2,21 @@
|
||||
#include <ccan/array_size/array_size.h>
|
||||
#include <common/coin_mvt.h>
|
||||
#include <common/json_param.h>
|
||||
#include <common/memleak.h>
|
||||
#include <common/node_id.h>
|
||||
#include <common/type_to_string.h>
|
||||
#include <plugins/bkpr/db.h>
|
||||
#include <plugins/libplugin.h>
|
||||
|
||||
#define CHAIN_MOVE "chain_mvt"
|
||||
#define CHANNEL_MOVE "channel_mvt"
|
||||
|
||||
/* The database that we store all the accounting data in */
|
||||
static struct db *db ;
|
||||
|
||||
// FIXME: make relative to directory we're loaded into
|
||||
static char *db_dsn = "sqlite3://accounts.sqlite3";
|
||||
|
||||
static struct command_result *json_list_balances(struct command *cmd,
|
||||
const char *buf,
|
||||
const jsmntok_t *params)
|
||||
@@ -321,7 +329,9 @@ static const struct plugin_command commands[] = {
|
||||
|
||||
static const char *init(struct plugin *p, const char *b, const jsmntok_t *t)
|
||||
{
|
||||
// FIXME set up database, run migrations
|
||||
// FIXME: pass in database DSN as an option??
|
||||
db = notleak(db_setup(p, p, db_dsn));
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
168
plugins/bkpr/db.c
Normal file
168
plugins/bkpr/db.c
Normal file
@@ -0,0 +1,168 @@
|
||||
#include "config.h"
|
||||
#include <ccan/array_size/array_size.h>
|
||||
#include <db/bindings.h>
|
||||
#include <db/common.h>
|
||||
#include <db/exec.h>
|
||||
#include <db/utils.h>
|
||||
#include <plugins/bkpr/db.h>
|
||||
#include <plugins/libplugin.h>
|
||||
#include <stdio.h>
|
||||
|
||||
struct migration {
|
||||
const char *sql;
|
||||
void (*func)(struct plugin *p, struct db *db);
|
||||
};
|
||||
|
||||
/* Do not reorder or remove elements from this array.
|
||||
* It is used to migrate existing databases from a prevoius state, based on
|
||||
* string indicies */
|
||||
static struct migration db_migrations[] = {
|
||||
{SQL("CREATE TABLE version (version INTEGER);"), NULL},
|
||||
{SQL("INSERT INTO version VALUES (1);"), NULL},
|
||||
{SQL("CREATE TABLE vars ("
|
||||
" name TEXT"
|
||||
", val TEXT"
|
||||
", intval INTEGER"
|
||||
", blobval BLOB"
|
||||
", PRIMARY KEY (name)"
|
||||
");"),
|
||||
NULL},
|
||||
{SQL("INSERT INTO vars ("
|
||||
" name"
|
||||
", intval"
|
||||
") VALUES ("
|
||||
" 'data_version'"
|
||||
", 0"
|
||||
");"),
|
||||
NULL},
|
||||
{SQL("CREATE TABLE accounts ("
|
||||
" id BIGSERIAL"
|
||||
", name TEXT"
|
||||
", peer_id BLOB"
|
||||
", opened_event_id BIGINT"
|
||||
", closed_event_id BIGINT"
|
||||
", onchain_resolved_block INTEGER"
|
||||
", is_wallet INTEGER"
|
||||
", we_opened INTEGER"
|
||||
", leased INTEGER"
|
||||
", last_updated_ts BIGINT"
|
||||
", last_updated_block INTEGER"
|
||||
", PRIMARY KEY (id)"
|
||||
");"),
|
||||
NULL},
|
||||
{SQL("CREATE TABLE chain_events ("
|
||||
" id BIGSERIAL"
|
||||
", account_id BIGINT REFERENCES accounts(id)"
|
||||
", tag TEXT"
|
||||
", credit BIGINT"
|
||||
", debit BIGINT"
|
||||
", output_value BIGINT"
|
||||
", currency TEXT"
|
||||
", timestamp BIGINT"
|
||||
", blockheight INTEGER"
|
||||
", utxo_txid BLOB"
|
||||
", outnum INTEGER"
|
||||
", spending_txid BLOB"
|
||||
", PRIMARY KEY (id)"
|
||||
");"),
|
||||
NULL},
|
||||
{SQL("CREATE TABLE channel_events ("
|
||||
" id BIGSERIAL"
|
||||
", account_id BIGINT REFERENCES accounts(id)"
|
||||
", tag TEXT"
|
||||
", credit BIGINT"
|
||||
", debit BIGINT"
|
||||
", fees BIGINT"
|
||||
", currency TEXT"
|
||||
", payment_id BLOB"
|
||||
", part_id INTEGER"
|
||||
", timestamp BIGINT"
|
||||
", PRIMARY KEY (id)"
|
||||
");"),
|
||||
NULL},
|
||||
{SQL("CREATE TABLE onchain_fees ("
|
||||
"account_id BIGINT REFERENCES accounts(id)"
|
||||
", txid BLOB"
|
||||
", amount BIGINT"
|
||||
", PRIMARY KEY (account_id, txid)"
|
||||
");"),
|
||||
NULL},
|
||||
};
|
||||
|
||||
static bool db_migrate(struct plugin *p, struct db *db)
|
||||
{
|
||||
/* Read current version from database */
|
||||
int current, orig, available;
|
||||
struct db_stmt *stmt;
|
||||
|
||||
orig = current = db_get_version(db);
|
||||
available = ARRAY_SIZE(db_migrations) - 1;
|
||||
|
||||
if (current == -1)
|
||||
plugin_log(p, LOG_INFORM, "Creating database");
|
||||
else if (available < current)
|
||||
plugin_err(p,
|
||||
"Refusing to migrate down from version %u to %u",
|
||||
current, available);
|
||||
else if (current != available)
|
||||
plugin_log(p, LOG_INFORM,
|
||||
"Updating database from version %u to %u",
|
||||
current, available);
|
||||
|
||||
while (current < available) {
|
||||
current++;
|
||||
if (db_migrations[current].sql) {
|
||||
struct db_stmt *stmt =
|
||||
db_prepare_v2(db, db_migrations[current].sql);
|
||||
db_exec_prepared_v2(take(stmt));
|
||||
}
|
||||
if (db_migrations[current].func)
|
||||
db_migrations[current].func(p, db);
|
||||
}
|
||||
|
||||
/* Finally, update the version number in the version table */
|
||||
stmt = db_prepare_v2(db, SQL("UPDATE version SET version=?;"));
|
||||
db_bind_int(stmt, 0, available);
|
||||
db_exec_prepared_v2(take(stmt));
|
||||
|
||||
return current != orig;
|
||||
}
|
||||
|
||||
/* Implement db_fatal, as a wrapper around fatal.
|
||||
* We use a ifndef block so that it can get be
|
||||
* implemented in a test file first, if necessary */
|
||||
#ifndef DB_FATAL
|
||||
#define DB_FATAL
|
||||
void db_fatal(const char *fmt, ...)
|
||||
{
|
||||
va_list ap;
|
||||
|
||||
va_start(ap, fmt);
|
||||
vfprintf(stderr, fmt, ap);
|
||||
fprintf(stderr, "\n");
|
||||
va_end(ap);
|
||||
|
||||
exit(1);
|
||||
}
|
||||
#endif /* DB_FATAL */
|
||||
|
||||
struct db *db_setup(const tal_t *ctx, struct plugin *p, char *db_dsn)
|
||||
{
|
||||
bool migrated;
|
||||
struct db *db = db_open(ctx, db_dsn);
|
||||
|
||||
db->report_changes_fn = NULL;
|
||||
|
||||
db_begin_transaction(db);
|
||||
migrated = db_migrate(p, db);
|
||||
db->data_version = db_data_version_get(db);
|
||||
db_commit_transaction(db);
|
||||
|
||||
/* This needs to be done outside a transaction, apparently.
|
||||
* It's a good idea to do this every so often, and on db
|
||||
* upgrade is a reasonable time. */
|
||||
if (migrated && !db->config->vacuum_fn(db))
|
||||
db_fatal("Error vacuuming db: %s", db->error);
|
||||
|
||||
return db;
|
||||
}
|
||||
11
plugins/bkpr/db.h
Normal file
11
plugins/bkpr/db.h
Normal file
@@ -0,0 +1,11 @@
|
||||
#ifndef LIGHTNING_PLUGINS_BKPR_DB_H
|
||||
#define LIGHTNING_PLUGINS_BKPR_DB_H
|
||||
#include "config.h"
|
||||
#include <ccan/tal/tal.h>
|
||||
|
||||
struct plugin;
|
||||
struct db;
|
||||
|
||||
struct db *db_setup(const tal_t *ctx, struct plugin *p, char *db_dsn);
|
||||
|
||||
#endif /* LIGHTNING_PLUGINS_BKPR_DB_H */
|
||||
Reference in New Issue
Block a user