From 60285187fd0f8faa66d637c0608b505d98a936d3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 24 Feb 2019 15:42:09 +1030 Subject: [PATCH] db: support more powerful migrations. Allow a function as well as (or instead of!) an sql statement. That will let us do things like set per-channel values to the global defaults, for example. Since we remove the NULL termination, the final entry is ARRAY_SIZE()-1 not ARRAY_SIZE()-2. Signed-off-by: Rusty Russell --- wallet/db.c | 684 ++++++++++++++++++++------------------- wallet/db.h | 4 +- wallet/test/run-db.c | 15 +- wallet/test/run-wallet.c | 2 +- wallet/wallet.c | 2 +- 5 files changed, 354 insertions(+), 353 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index 314a6e2a8..bf12a44d5 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -15,13 +15,18 @@ #define db_fatal fatal #endif +struct migration { + const char *sql; + void (*func)(struct lightningd *ld, struct db *db); +}; + /* Do not reorder or remove elements from this array, it is used to * migrate existing databases from a previous state, based on the * string indices */ -char *dbmigrations[] = { - "CREATE TABLE version (version INTEGER)", - "INSERT INTO version VALUES (1)", - "CREATE TABLE outputs ( \ +static struct migration dbmigrations[] = { + { "CREATE TABLE version (version INTEGER)", NULL }, + { "INSERT INTO version VALUES (1)", NULL }, + { "CREATE TABLE outputs ( \ prev_out_tx CHAR(64), \ prev_out_index INTEGER, \ value INTEGER, \ @@ -29,341 +34,339 @@ char *dbmigrations[] = { status INTEGER, \ keyindex INTEGER, \ PRIMARY KEY (prev_out_tx, prev_out_index) \ - );", - "CREATE TABLE vars (name VARCHAR(32), val VARCHAR(255), PRIMARY KEY (name));", - "CREATE TABLE shachains ( \ + );", NULL }, + { "CREATE TABLE vars (name VARCHAR(32), val VARCHAR(255), PRIMARY KEY (name));", NULL }, + { "CREATE TABLE shachains ( \ id INTEGER, \ min_index INTEGER, \ num_valid INTEGER, \ - PRIMARY KEY (id));", - "CREATE TABLE shachain_known ( \ + PRIMARY KEY (id));", NULL }, + { "CREATE TABLE shachain_known ( \ shachain_id INTEGER REFERENCES shachains(id) ON DELETE CASCADE, \ pos INTEGER, \ idx INTEGER, \ hash BLOB, \ - PRIMARY KEY (shachain_id, pos));", - "CREATE TABLE channels (" - " id INTEGER," /* chan->id */ - " peer_id INTEGER REFERENCES peers(id) ON DELETE CASCADE," - " short_channel_id BLOB," - " channel_config_local INTEGER," - " channel_config_remote INTEGER," - " state INTEGER," - " funder INTEGER," - " channel_flags INTEGER," - " minimum_depth INTEGER," - " next_index_local INTEGER," - " next_index_remote INTEGER," - " next_htlc_id INTEGER, " - " funding_tx_id BLOB," - " funding_tx_outnum INTEGER," - " funding_satoshi INTEGER," - " funding_locked_remote INTEGER," - " push_msatoshi INTEGER," - " msatoshi_local INTEGER," /* our_msatoshi */ - /* START channel_info */ - " fundingkey_remote BLOB," - " revocation_basepoint_remote BLOB," - " payment_basepoint_remote BLOB," - " htlc_basepoint_remote BLOB," - " delayed_payment_basepoint_remote BLOB," - " per_commit_remote BLOB," - " old_per_commit_remote BLOB," - " local_feerate_per_kw INTEGER," - " remote_feerate_per_kw INTEGER," - /* END channel_info */ - " shachain_remote_id INTEGER," - " shutdown_scriptpubkey_remote BLOB," - " shutdown_keyidx_local INTEGER," - " last_sent_commit_state INTEGER," - " last_sent_commit_id INTEGER," - " last_tx BLOB," - " last_sig BLOB," - " closing_fee_received INTEGER," - " closing_sig_received BLOB," - " PRIMARY KEY (id)" - ");", - "CREATE TABLE peers (" - " id INTEGER," - " node_id BLOB UNIQUE," /* pubkey */ - " address TEXT," - " PRIMARY KEY (id)" - ");", - "CREATE TABLE channel_configs (" - " id INTEGER," - " dust_limit_satoshis INTEGER," - " max_htlc_value_in_flight_msat INTEGER," - " channel_reserve_satoshis INTEGER," - " htlc_minimum_msat INTEGER," - " to_self_delay INTEGER," - " max_accepted_htlcs INTEGER," - " PRIMARY KEY (id)" - ");", - "CREATE TABLE channel_htlcs (" - " id INTEGER," - " channel_id INTEGER REFERENCES channels(id) ON DELETE CASCADE," - " channel_htlc_id INTEGER," - " direction INTEGER," - " origin_htlc INTEGER," - " msatoshi INTEGER," - " cltv_expiry INTEGER," - " payment_hash BLOB," - " payment_key BLOB," - " routing_onion BLOB," - " failuremsg BLOB," - " malformed_onion INTEGER," - " hstate INTEGER," - " shared_secret BLOB," - " PRIMARY KEY (id)," - " UNIQUE (channel_id, channel_htlc_id, direction)" - ");", - "CREATE TABLE invoices (" - " id INTEGER," - " state INTEGER," - " msatoshi INTEGER," - " payment_hash BLOB," - " payment_key BLOB," - " label TEXT," - " PRIMARY KEY (id)," - " UNIQUE (label)," - " UNIQUE (payment_hash)" - ");", - "CREATE TABLE payments (" - " id INTEGER," - " timestamp INTEGER," - " status INTEGER," - " payment_hash BLOB," - " direction INTEGER," - " destination BLOB," - " msatoshi INTEGER," - " PRIMARY KEY (id)," - " UNIQUE (payment_hash)" - ");", - /* Add expiry field to invoices (effectively infinite). */ - "ALTER TABLE invoices ADD expiry_time INTEGER;", - "UPDATE invoices SET expiry_time=9223372036854775807;", - /* Add pay_index field to paid invoices (initially, same order as id). */ - "ALTER TABLE invoices ADD pay_index INTEGER;", - "CREATE UNIQUE INDEX invoices_pay_index" - " ON invoices(pay_index);", - "UPDATE invoices SET pay_index=id WHERE state=1;", /* only paid invoice */ - /* Create next_pay_index variable (highest pay_index). */ - "INSERT OR REPLACE INTO vars(name, val)" - " VALUES('next_pay_index', " - " COALESCE((SELECT MAX(pay_index) FROM invoices WHERE state=1), 0) + 1" - " );", - /* Create first_block field; initialize from channel id if any. - * This fails for channels still awaiting lockin, but that only applies to - * pre-release software, so it's forgivable. */ - "ALTER TABLE channels ADD first_blocknum INTEGER;", - "UPDATE channels SET first_blocknum=CAST(short_channel_id AS INTEGER) WHERE short_channel_id IS NOT NULL;", - "ALTER TABLE outputs ADD COLUMN channel_id INTEGER;", - "ALTER TABLE outputs ADD COLUMN peer_id BLOB;", - "ALTER TABLE outputs ADD COLUMN commitment_point BLOB;", - "ALTER TABLE invoices ADD COLUMN msatoshi_received INTEGER;", - /* Normally impossible, so at least we'll know if databases are ancient. */ - "UPDATE invoices SET msatoshi_received=0 WHERE state=1;", - "ALTER TABLE channels ADD COLUMN last_was_revoke INTEGER;", - /* We no longer record incoming payments: invoices cover that. - * Without ALTER_TABLE DROP COLUMN support we need to do this by - * rename & copy, which works because there are no triggers etc. */ - "ALTER TABLE payments RENAME TO temp_payments;", - "CREATE TABLE payments (" - " id INTEGER," - " timestamp INTEGER," - " status INTEGER," - " payment_hash BLOB," - " destination BLOB," - " msatoshi INTEGER," - " PRIMARY KEY (id)," - " UNIQUE (payment_hash)" - ");", - "INSERT INTO payments SELECT id, timestamp, status, payment_hash, destination, msatoshi FROM temp_payments WHERE direction=1;", - "DROP TABLE temp_payments;", - /* We need to keep the preimage in case they ask to pay again. */ - "ALTER TABLE payments ADD COLUMN payment_preimage BLOB;", - /* We need to keep the shared secrets to decode error returns. */ - "ALTER TABLE payments ADD COLUMN path_secrets BLOB;", - /* Create time-of-payment of invoice, default already-paid - * invoices to current time. */ - "ALTER TABLE invoices ADD paid_timestamp INTEGER;", - "UPDATE invoices" - " SET paid_timestamp = strftime('%s', 'now')" - " WHERE state = 1;", - /* We need to keep the route node pubkeys and short channel ids to - * correctly mark routing failures. We separate short channel ids - * because we cannot safely save them as blobs due to byteorder - * concerns. */ - "ALTER TABLE payments ADD COLUMN route_nodes BLOB;", - "ALTER TABLE payments ADD COLUMN route_channels TEXT;", - "CREATE TABLE htlc_sigs (channelid INTEGER REFERENCES channels(id) ON DELETE CASCADE, signature BLOB);", - "CREATE INDEX channel_idx ON htlc_sigs (channelid)", - /* Get rid of OPENINGD entries; we don't put them in db any more */ - "DELETE FROM channels WHERE state=1", - /* Keep track of db upgrades, for debugging */ - "CREATE TABLE db_upgrades (upgrade_from INTEGER, lightning_version TEXT);", - /* We used not to clean up peers when their channels were gone. */ - "DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);", - /* The ONCHAIND_CHEATED/THEIR_UNILATERAL/OUR_UNILATERAL/MUTUAL are now one */ - "UPDATE channels SET STATE = 8 WHERE state > 8;", - /* Add bolt11 to invoices table*/ - "ALTER TABLE invoices ADD bolt11 TEXT;", - /* What do we think the head of the blockchain looks like? Used - * primarily to track confirmations across restarts and making - * sure we handle reorgs correctly. */ - "CREATE TABLE blocks (height INT, hash BLOB, prev_hash BLOB, UNIQUE(height));", - /* ON DELETE CASCADE would have been nice for confirmation_height, - * so that we automatically delete outputs that fall off the - * blockchain and then we rediscover them if they are included - * again. However, we have the their_unilateral/to_us which we - * can't simply recognize from the chain without additional - * hints. So we just mark them as unconfirmed should the block - * die. */ - "ALTER TABLE outputs ADD COLUMN confirmation_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;", - "ALTER TABLE outputs ADD COLUMN spend_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;", - /* Create a covering index that covers both fields */ - "CREATE INDEX output_height_idx ON outputs (confirmation_height, spend_height);", - "CREATE TABLE utxoset (" - " txid BLOB," - " outnum INT," - " blockheight INT REFERENCES blocks(height) ON DELETE CASCADE," - " spendheight INT REFERENCES blocks(height) ON DELETE SET NULL," - " txindex INT," - " scriptpubkey BLOB," - " satoshis BIGINT," - " PRIMARY KEY(txid, outnum));", - "CREATE INDEX short_channel_id ON utxoset (blockheight, txindex, outnum)", - /* Necessary index for long rollbacks of the blockchain, otherwise we're - * doing table scans for every block removed. */ - "CREATE INDEX utxoset_spend ON utxoset (spendheight)", - /* Assign key 0 to unassigned shutdown_keyidx_local. */ - "UPDATE channels SET shutdown_keyidx_local=0 WHERE shutdown_keyidx_local = -1;", - /* FIXME: We should rename shutdown_keyidx_local to final_key_index */ - /* -- Payment routing failure information -- */ - /* BLOB if failure was due to unparseable onion, NULL otherwise */ - "ALTER TABLE payments ADD failonionreply BLOB;", - /* 0 if we could theoretically retry, 1 if PERM fail at payee */ - "ALTER TABLE payments ADD faildestperm INTEGER;", - /* Contents of routing_failure (only if not unparseable onion) */ - "ALTER TABLE payments ADD failindex INTEGER;", /* erring_index */ - "ALTER TABLE payments ADD failcode INTEGER;", /* failcode */ - "ALTER TABLE payments ADD failnode BLOB;", /* erring_node */ - "ALTER TABLE payments ADD failchannel BLOB;", /* erring_channel */ - "ALTER TABLE payments ADD failupdate BLOB;", /* channel_update - can be NULL*/ - /* -- Payment routing failure information ends -- */ - /* Delete route data for already succeeded or failed payments */ - "UPDATE payments" - " SET path_secrets = NULL" - " , route_nodes = NULL" - " , route_channels = NULL" - " WHERE status <> 0;", /* PAYMENT_PENDING */ - /* -- Routing statistics -- */ - "ALTER TABLE channels ADD in_payments_offered INTEGER;", - "ALTER TABLE channels ADD in_payments_fulfilled INTEGER;", - "ALTER TABLE channels ADD in_msatoshi_offered INTEGER;", - "ALTER TABLE channels ADD in_msatoshi_fulfilled INTEGER;", - "ALTER TABLE channels ADD out_payments_offered INTEGER;", - "ALTER TABLE channels ADD out_payments_fulfilled INTEGER;", - "ALTER TABLE channels ADD out_msatoshi_offered INTEGER;", - "ALTER TABLE channels ADD out_msatoshi_fulfilled INTEGER;", - "UPDATE channels" - " SET in_payments_offered = 0, in_payments_fulfilled = 0" - " , in_msatoshi_offered = 0, in_msatoshi_fulfilled = 0" - " , out_payments_offered = 0, out_payments_fulfilled = 0" - " , out_msatoshi_offered = 0, out_msatoshi_fulfilled = 0" - " ;", - /* -- Routing statistics ends --*/ - /* Record the msatoshi actually sent in a payment. */ - "ALTER TABLE payments ADD msatoshi_sent INTEGER;", - "UPDATE payments SET msatoshi_sent = msatoshi;", - /* Delete dangling utxoset entries due to Issue #1280 */ - "DELETE FROM utxoset WHERE blockheight IN (" - " SELECT DISTINCT(blockheight)" - " FROM utxoset LEFT OUTER JOIN blocks on (blockheight == blocks.height) " - " WHERE blocks.hash IS NULL" - ");", - /* Record feerate range, to optimize onchaind grinding actual fees. */ - "ALTER TABLE channels ADD min_possible_feerate INTEGER;", - "ALTER TABLE channels ADD max_possible_feerate INTEGER;", - /* https://bitcoinfees.github.io/#1d says Dec 17 peak was ~1M sat/kb - * which is 250,000 sat/Sipa */ - "UPDATE channels SET min_possible_feerate=0, max_possible_feerate=250000;", - /* -- Min and max msatoshi_to_us -- */ - "ALTER TABLE channels ADD msatoshi_to_us_min INTEGER;", - "ALTER TABLE channels ADD msatoshi_to_us_max INTEGER;", - "UPDATE channels" - " SET msatoshi_to_us_min = msatoshi_local" - " , msatoshi_to_us_max = msatoshi_local" - " ;", - /* -- Min and max msatoshi_to_us ends -- */ - /* Transactions we are interested in. Either we sent them ourselves or we - * are watching them. We don't cascade block height deletes so we don't - * forget any of them by accident.*/ - "CREATE TABLE transactions (" - " id BLOB" - ", blockheight INTEGER REFERENCES blocks(height) ON DELETE SET NULL" - ", txindex INTEGER" - ", rawtx BLOB" - ", PRIMARY KEY (id)" - ");", - /* -- Detailed payment failure -- */ - "ALTER TABLE payments ADD faildetail TEXT;", - "UPDATE payments" - " SET faildetail = 'unspecified payment failure reason'" - " WHERE status = 2;", /* PAYMENT_FAILED */ - /* -- Detailed payment faiure ends -- */ - "CREATE TABLE channeltxs (" - /* The id serves as insertion order and short ID */ - " id INTEGER" - ", channel_id INTEGER REFERENCES channels(id) ON DELETE CASCADE" - ", type INTEGER" - ", transaction_id BLOB REFERENCES transactions(id) ON DELETE CASCADE" - /* The input_num is only used by the txo_watch, 0 if txwatch */ - ", input_num INTEGER" - /* The height at which we sent the depth notice */ - ", blockheight INTEGER REFERENCES blocks(height) ON DELETE CASCADE" - ", PRIMARY KEY(id)" - ");", - /* -- Set the correct rescan height for PR #1398 -- */ - /* Delete blocks that are higher than our initial scan point, this is a - * no-op if we don't have a channel. */ - "DELETE FROM blocks WHERE height > (SELECT MIN(first_blocknum) FROM channels);", - /* Now make sure we have the lower bound block with the first_blocknum - * height. This may introduce a block with NULL height if we didn't have any - * blocks, remove that in the next. */ - "INSERT OR IGNORE INTO blocks (height) VALUES ((SELECT MIN(first_blocknum) FROM channels));", - "DELETE FROM blocks WHERE height IS NULL;", - /* -- End of PR #1398 -- */ - "ALTER TABLE invoices ADD description TEXT;", - /* FIXME: payments table 'description' is really a 'label' */ - "ALTER TABLE payments ADD description TEXT;", - /* future_per_commitment_point if other side proves we're out of date -- */ - "ALTER TABLE channels ADD future_per_commitment_point BLOB;", - /* last_sent_commit array fix */ - "ALTER TABLE channels ADD last_sent_commit BLOB;", - - /* Stats table to track forwarded HTLCs. The values in the HTLCs - * and their states are replicated here and the entries are not - * deleted when the HTLC entries or the channel entries are - * deleted to avoid unexpected drops in statistics. */ - "CREATE TABLE forwarded_payments (" - " in_htlc_id INTEGER REFERENCES channel_htlcs(id) ON DELETE SET NULL" - ", out_htlc_id INTEGER REFERENCES channel_htlcs(id) ON DELETE SET NULL" - ", in_channel_scid INTEGER" - ", out_channel_scid INTEGER" - ", in_msatoshi INTEGER" - ", out_msatoshi INTEGER" - ", state INTEGER" - ", UNIQUE(in_htlc_id, out_htlc_id)" - ");", - /* Add a direction for failed payments. */ - "ALTER TABLE payments ADD faildirection INTEGER;", /* erring_direction */ - /* Fix dangling peers with no channels. */ - "DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);", - "ALTER TABLE outputs ADD scriptpubkey BLOB;", - /* Keep bolt11 string for payments. */ - "ALTER TABLE payments ADD bolt11 TEXT;", - NULL, + PRIMARY KEY (shachain_id, pos));", NULL }, + { "CREATE TABLE channels (" + " id INTEGER," /* chan->id */ + " peer_id INTEGER REFERENCES peers(id) ON DELETE CASCADE," + " short_channel_id BLOB," + " channel_config_local INTEGER," + " channel_config_remote INTEGER," + " state INTEGER," + " funder INTEGER," + " channel_flags INTEGER," + " minimum_depth INTEGER," + " next_index_local INTEGER," + " next_index_remote INTEGER," + " next_htlc_id INTEGER, " + " funding_tx_id BLOB," + " funding_tx_outnum INTEGER," + " funding_satoshi INTEGER," + " funding_locked_remote INTEGER," + " push_msatoshi INTEGER," + " msatoshi_local INTEGER," /* our_msatoshi */ + /* START channel_info */ + " fundingkey_remote BLOB," + " revocation_basepoint_remote BLOB," + " payment_basepoint_remote BLOB," + " htlc_basepoint_remote BLOB," + " delayed_payment_basepoint_remote BLOB," + " per_commit_remote BLOB," + " old_per_commit_remote BLOB," + " local_feerate_per_kw INTEGER," + " remote_feerate_per_kw INTEGER," + /* END channel_info */ + " shachain_remote_id INTEGER," + " shutdown_scriptpubkey_remote BLOB," + " shutdown_keyidx_local INTEGER," + " last_sent_commit_state INTEGER," + " last_sent_commit_id INTEGER," + " last_tx BLOB," + " last_sig BLOB," + " closing_fee_received INTEGER," + " closing_sig_received BLOB," + " PRIMARY KEY (id)" + ");", NULL }, + { "CREATE TABLE peers (" + " id INTEGER," + " node_id BLOB UNIQUE," /* pubkey */ + " address TEXT," + " PRIMARY KEY (id)" + ");", NULL }, + { "CREATE TABLE channel_configs (" + " id INTEGER," + " dust_limit_satoshis INTEGER," + " max_htlc_value_in_flight_msat INTEGER," + " channel_reserve_satoshis INTEGER," + " htlc_minimum_msat INTEGER," + " to_self_delay INTEGER," + " max_accepted_htlcs INTEGER," + " PRIMARY KEY (id)" + ");", NULL }, + { "CREATE TABLE channel_htlcs (" + " id INTEGER," + " channel_id INTEGER REFERENCES channels(id) ON DELETE CASCADE," + " channel_htlc_id INTEGER," + " direction INTEGER," + " origin_htlc INTEGER," + " msatoshi INTEGER," + " cltv_expiry INTEGER," + " payment_hash BLOB," + " payment_key BLOB," + " routing_onion BLOB," + " failuremsg BLOB," + " malformed_onion INTEGER," + " hstate INTEGER," + " shared_secret BLOB," + " PRIMARY KEY (id)," + " UNIQUE (channel_id, channel_htlc_id, direction)" + ");", NULL }, + { "CREATE TABLE invoices (" + " id INTEGER," + " state INTEGER," + " msatoshi INTEGER," + " payment_hash BLOB," + " payment_key BLOB," + " label TEXT," + " PRIMARY KEY (id)," + " UNIQUE (label)," + " UNIQUE (payment_hash)" + ");", NULL }, + { "CREATE TABLE payments (" + " id INTEGER," + " timestamp INTEGER," + " status INTEGER," + " payment_hash BLOB," + " direction INTEGER," + " destination BLOB," + " msatoshi INTEGER," + " PRIMARY KEY (id)," + " UNIQUE (payment_hash)" + ");", NULL }, + /* Add expiry field to invoices (effectively infinite). */ + { "ALTER TABLE invoices ADD expiry_time INTEGER;", NULL }, + { "UPDATE invoices SET expiry_time=9223372036854775807;", NULL }, + /* Add pay_index field to paid invoices (initially, same order as id). */ + { "ALTER TABLE invoices ADD pay_index INTEGER;", NULL }, + { "CREATE UNIQUE INDEX invoices_pay_index" + " ON invoices(pay_index);", NULL }, + { "UPDATE invoices SET pay_index=id WHERE state=1;", NULL }, /* only paid invoice */ + /* Create next_pay_index variable (highest pay_index). */ + { "INSERT OR REPLACE INTO vars(name, val)" + " VALUES('next_pay_index', " + " COALESCE((SELECT MAX(pay_index) FROM invoices WHERE state=1), 0) + 1" + " );", NULL }, + /* Create first_block field; initialize from channel id if any. + * This fails for channels still awaiting lockin, but that only applies to + * pre-release software, so it's forgivable. */ + { "ALTER TABLE channels ADD first_blocknum INTEGER;", NULL }, + { "UPDATE channels SET first_blocknum=CAST(short_channel_id AS INTEGER) WHERE short_channel_id IS NOT NULL;", NULL }, + { "ALTER TABLE outputs ADD COLUMN channel_id INTEGER;", NULL }, + { "ALTER TABLE outputs ADD COLUMN peer_id BLOB;", NULL }, + { "ALTER TABLE outputs ADD COLUMN commitment_point BLOB;", NULL }, + { "ALTER TABLE invoices ADD COLUMN msatoshi_received INTEGER;", NULL }, + /* Normally impossible, so at least we'll know if databases are ancient. */ + { "UPDATE invoices SET msatoshi_received=0 WHERE state=1;", NULL }, + { "ALTER TABLE channels ADD COLUMN last_was_revoke INTEGER;", NULL }, + /* We no longer record incoming payments: invoices cover that. + * Without ALTER_TABLE DROP COLUMN support we need to do this by + * rename & copy, which works because there are no triggers etc. */ + { "ALTER TABLE payments RENAME TO temp_payments;", NULL }, + { "CREATE TABLE payments (" + " id INTEGER," + " timestamp INTEGER," + " status INTEGER," + " payment_hash BLOB," + " destination BLOB," + " msatoshi INTEGER," + " PRIMARY KEY (id)," + " UNIQUE (payment_hash)" + ");", NULL }, + { "INSERT INTO payments SELECT id, timestamp, status, payment_hash, destination, msatoshi FROM temp_payments WHERE direction=1;", NULL }, + { "DROP TABLE temp_payments;", NULL }, + /* We need to keep the preimage in case they ask to pay again. */ + { "ALTER TABLE payments ADD COLUMN payment_preimage BLOB;", NULL }, + /* We need to keep the shared secrets to decode error returns. */ + { "ALTER TABLE payments ADD COLUMN path_secrets BLOB;", NULL }, + /* Create time-of-payment of invoice, default already-paid + * invoices to current time. */ + { "ALTER TABLE invoices ADD paid_timestamp INTEGER;", NULL }, + { "UPDATE invoices" + " SET paid_timestamp = strftime('%s', 'now')" + " WHERE state = 1;", NULL }, + /* We need to keep the route node pubkeys and short channel ids to + * correctly mark routing failures. We separate short channel ids + * because we cannot safely save them as blobs due to byteorder + * concerns. */ + { "ALTER TABLE payments ADD COLUMN route_nodes BLOB;", NULL }, + { "ALTER TABLE payments ADD COLUMN route_channels TEXT;", NULL }, + { "CREATE TABLE htlc_sigs (channelid INTEGER REFERENCES channels(id) ON DELETE CASCADE, signature BLOB);", NULL }, + { "CREATE INDEX channel_idx ON htlc_sigs (channelid)", NULL }, + /* Get rid of OPENINGD entries; we don't put them in db any more */ + { "DELETE FROM channels WHERE state=1", NULL }, + /* Keep track of db upgrades, for debugging */ + { "CREATE TABLE db_upgrades (upgrade_from INTEGER, lightning_version TEXT);", NULL }, + /* We used not to clean up peers when their channels were gone. */ + { "DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);", NULL }, + /* The ONCHAIND_CHEATED/THEIR_UNILATERAL/OUR_UNILATERAL/MUTUAL are now one */ + { "UPDATE channels SET STATE = 8 WHERE state > 8;", NULL }, + /* Add bolt11 to invoices table*/ + { "ALTER TABLE invoices ADD bolt11 TEXT;", NULL }, + /* What do we think the head of the blockchain looks like? Used + * primarily to track confirmations across restarts and making + * sure we handle reorgs correctly. */ + { "CREATE TABLE blocks (height INT, hash BLOB, prev_hash BLOB, UNIQUE(height));", NULL }, + /* ON DELETE CASCADE would have been nice for confirmation_height, + * so that we automatically delete outputs that fall off the + * blockchain and then we rediscover them if they are included + * again. However, we have the their_unilateral/to_us which we + * can't simply recognize from the chain without additional + * hints. So we just mark them as unconfirmed should the block + * die. */ + { "ALTER TABLE outputs ADD COLUMN confirmation_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;", NULL }, + { "ALTER TABLE outputs ADD COLUMN spend_height INTEGER REFERENCES blocks(height) ON DELETE SET NULL;", NULL }, + /* Create a covering index that covers both fields */ + { "CREATE INDEX output_height_idx ON outputs (confirmation_height, spend_height);", NULL }, + { "CREATE TABLE utxoset (" + " txid BLOB," + " outnum INT," + " blockheight INT REFERENCES blocks(height) ON DELETE CASCADE," + " spendheight INT REFERENCES blocks(height) ON DELETE SET NULL," + " txindex INT," + " scriptpubkey BLOB," + " satoshis BIGINT," + " PRIMARY KEY(txid, outnum));", NULL }, + { "CREATE INDEX short_channel_id ON utxoset (blockheight, txindex, outnum)", NULL }, + /* Necessary index for long rollbacks of the blockchain, otherwise we're + * doing table scans for every block removed. */ + { "CREATE INDEX utxoset_spend ON utxoset (spendheight)", NULL }, + /* Assign key 0 to unassigned shutdown_keyidx_local. */ + { "UPDATE channels SET shutdown_keyidx_local=0 WHERE shutdown_keyidx_local = -1;", NULL }, + /* FIXME: We should rename shutdown_keyidx_local to final_key_index */ + /* -- Payment routing failure information -- */ + /* BLOB if failure was due to unparseable onion, NULL otherwise */ + { "ALTER TABLE payments ADD failonionreply BLOB;", NULL }, + /* 0 if we could theoretically retry, 1 if PERM fail at payee */ + { "ALTER TABLE payments ADD faildestperm INTEGER;", NULL }, + /* Contents of routing_failure (only if not unparseable onion) */ + { "ALTER TABLE payments ADD failindex INTEGER;", NULL }, /* erring_index */ + { "ALTER TABLE payments ADD failcode INTEGER;", NULL }, /* failcode */ + { "ALTER TABLE payments ADD failnode BLOB;", NULL }, /* erring_node */ + { "ALTER TABLE payments ADD failchannel BLOB;", NULL }, /* erring_channel */ + { "ALTER TABLE payments ADD failupdate BLOB;", NULL }, /* channel_update - can be NULL*/ + /* -- Payment routing failure information ends -- */ + /* Delete route data for already succeeded or failed payments */ + { "UPDATE payments" + " SET path_secrets = NULL" + " , route_nodes = NULL" + " , route_channels = NULL" + " WHERE status <> 0;", NULL }, /* PAYMENT_PENDING */ + /* -- Routing statistics -- */ + { "ALTER TABLE channels ADD in_payments_offered INTEGER;", NULL }, + { "ALTER TABLE channels ADD in_payments_fulfilled INTEGER;", NULL }, + { "ALTER TABLE channels ADD in_msatoshi_offered INTEGER;", NULL }, + { "ALTER TABLE channels ADD in_msatoshi_fulfilled INTEGER;", NULL }, + { "ALTER TABLE channels ADD out_payments_offered INTEGER;", NULL }, + { "ALTER TABLE channels ADD out_payments_fulfilled INTEGER;", NULL }, + { "ALTER TABLE channels ADD out_msatoshi_offered INTEGER;", NULL }, + { "ALTER TABLE channels ADD out_msatoshi_fulfilled INTEGER;", NULL }, + { "UPDATE channels" + " SET in_payments_offered = 0, in_payments_fulfilled = 0" + " , in_msatoshi_offered = 0, in_msatoshi_fulfilled = 0" + " , out_payments_offered = 0, out_payments_fulfilled = 0" + " , out_msatoshi_offered = 0, out_msatoshi_fulfilled = 0" + " ;", NULL }, + /* -- Routing statistics ends --*/ + /* Record the msatoshi actually sent in a payment. */ + { "ALTER TABLE payments ADD msatoshi_sent INTEGER;", NULL }, + { "UPDATE payments SET msatoshi_sent = msatoshi;", NULL }, + /* Delete dangling utxoset entries due to Issue #1280 */ + { "DELETE FROM utxoset WHERE blockheight IN (" + " SELECT DISTINCT(blockheight)" + " FROM utxoset LEFT OUTER JOIN blocks on (blockheight == blocks.height) " + " WHERE blocks.hash IS NULL" + ");", NULL }, + /* Record feerate range, to optimize onchaind grinding actual fees. */ + { "ALTER TABLE channels ADD min_possible_feerate INTEGER;", NULL }, + { "ALTER TABLE channels ADD max_possible_feerate INTEGER;", NULL }, + /* https://bitcoinfees.github.io/#1d says Dec 17 peak was ~1M sat/kb + * which is 250,000 sat/Sipa */ + { "UPDATE channels SET min_possible_feerate=0, max_possible_feerate=250000;", NULL }, + /* -- Min and max msatoshi_to_us -- */ + { "ALTER TABLE channels ADD msatoshi_to_us_min INTEGER;", NULL }, + { "ALTER TABLE channels ADD msatoshi_to_us_max INTEGER;", NULL }, + { "UPDATE channels" + " SET msatoshi_to_us_min = msatoshi_local" + " , msatoshi_to_us_max = msatoshi_local" + " ;", NULL }, + /* -- Min and max msatoshi_to_us ends -- */ + /* Transactions we are interested in. Either we sent them ourselves or we + * are watching them. We don't cascade block height deletes so we don't + * forget any of them by accident.*/ + { "CREATE TABLE transactions (" + " id BLOB" + ", blockheight INTEGER REFERENCES blocks(height) ON DELETE SET NULL" + ", txindex INTEGER" + ", rawtx BLOB" + ", PRIMARY KEY (id)" + ");", NULL }, + /* -- Detailed payment failure -- */ + { "ALTER TABLE payments ADD faildetail TEXT;", NULL }, + { "UPDATE payments" + " SET faildetail = 'unspecified payment failure reason'" + " WHERE status = 2;", NULL }, /* PAYMENT_FAILED */ + /* -- Detailed payment faiure ends -- */ + { "CREATE TABLE channeltxs (" + /* The id serves as insertion order and short ID */ + " id INTEGER" + ", channel_id INTEGER REFERENCES channels(id) ON DELETE CASCADE" + ", type INTEGER" + ", transaction_id BLOB REFERENCES transactions(id) ON DELETE CASCADE" + /* The input_num is only used by the txo_watch, 0 if txwatch */ + ", input_num INTEGER" + /* The height at which we sent the depth notice */ + ", blockheight INTEGER REFERENCES blocks(height) ON DELETE CASCADE" + ", PRIMARY KEY(id)" + ");", NULL }, + /* -- Set the correct rescan height for PR #1398 -- */ + /* Delete blocks that are higher than our initial scan point, this is a + * no-op if we don't have a channel. */ + { "DELETE FROM blocks WHERE height > (SELECT MIN(first_blocknum) FROM channels);", NULL }, + /* Now make sure we have the lower bound block with the first_blocknum + * height. This may introduce a block with NULL height if we didn't have any + * blocks, remove that in the next. */ + { "INSERT OR IGNORE INTO blocks (height) VALUES ((SELECT MIN(first_blocknum) FROM channels));", NULL }, + { "DELETE FROM blocks WHERE height IS NULL;", NULL }, + /* -- End of PR #1398 -- */ + { "ALTER TABLE invoices ADD description TEXT;", NULL }, + /* FIXME: payments table 'description' is really a 'label' */ + { "ALTER TABLE payments ADD description TEXT;", NULL }, + /* future_per_commitment_point if other side proves we're out of date -- */ + { "ALTER TABLE channels ADD future_per_commitment_point BLOB;", NULL }, + /* last_sent_commit array fix */ + { "ALTER TABLE channels ADD last_sent_commit BLOB;", NULL }, + /* Stats table to track forwarded HTLCs. The values in the HTLCs + * and their states are replicated here and the entries are not + * deleted when the HTLC entries or the channel entries are + * deleted to avoid unexpected drops in statistics. */ + { "CREATE TABLE forwarded_payments (" + " in_htlc_id INTEGER REFERENCES channel_htlcs(id) ON DELETE SET NULL" + ", out_htlc_id INTEGER REFERENCES channel_htlcs(id) ON DELETE SET NULL" + ", in_channel_scid INTEGER" + ", out_channel_scid INTEGER" + ", in_msatoshi INTEGER" + ", out_msatoshi INTEGER" + ", state INTEGER" + ", UNIQUE(in_htlc_id, out_htlc_id)" + ");", NULL }, + /* Add a direction for failed payments. */ + { "ALTER TABLE payments ADD faildirection INTEGER;", NULL }, /* erring_direction */ + /* Fix dangling peers with no channels. */ + { "DELETE FROM peers WHERE id NOT IN (SELECT peer_id FROM channels);", NULL }, + { "ALTER TABLE outputs ADD scriptpubkey BLOB;", NULL }, + /* Keep bolt11 string for payments. */ + { "ALTER TABLE payments ADD bolt11 TEXT;", NULL }, }; /* Leak tracking. */ @@ -604,7 +607,7 @@ static int db_get_version(struct db *db) /** * db_migrate - Apply all remaining migrations from the current version */ -static void db_migrate(struct db *db, struct log *log) +static void db_migrate(struct lightningd *ld, struct db *db, struct log *log) { /* Attempt to read the version from the database */ int current, orig, available; @@ -612,7 +615,7 @@ static void db_migrate(struct db *db, struct log *log) db_begin_transaction(db); orig = current = db_get_version(db); - available = ARRAY_SIZE(dbmigrations) - 2; + available = ARRAY_SIZE(dbmigrations) - 1; if (current == -1) log_info(log, "Creating database"); @@ -623,8 +626,13 @@ static void db_migrate(struct db *db, struct log *log) log_info(log, "Updating database from version %u to %u", current, available); - while (current < available) - db_exec(__func__, db, "%s", dbmigrations[++current]); + while (current < available) { + current++; + if (dbmigrations[current].sql) + db_exec(__func__, db, "%s", dbmigrations[current].sql); + if (dbmigrations[current].func) + dbmigrations[current].func(ld, db); + } /* Finally update the version number in the version table */ db_exec(__func__, db, "UPDATE version SET version=%d;", available); @@ -638,11 +646,11 @@ static void db_migrate(struct db *db, struct log *log) db_commit_transaction(db); } -struct db *db_setup(const tal_t *ctx, struct log *log) +struct db *db_setup(const tal_t *ctx, struct lightningd *ld, struct log *log) { struct db *db = db_open(ctx, DB_FILE); - db_migrate(db, log); + db_migrate(ld, db, log); return db; } diff --git a/wallet/db.h b/wallet/db.h index 388394066..82f1e76aa 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -13,6 +13,7 @@ #include #include +struct lightningd; struct log; struct db { @@ -30,9 +31,10 @@ struct db { * * Params: * @ctx: the tal_t context to allocate from + * @ld: the lightningd context to hand to ugprade functions. * @log: where to log messages to */ -struct db *db_setup(const tal_t *ctx, struct log *log); +struct db *db_setup(const tal_t *ctx, struct lightningd *ld, struct log *log); /** * db_query - Prepare and execute a query, and return the result (or NULL) diff --git a/wallet/test/run-db.c b/wallet/test/run-db.c index fb80ee307..cb93ca8af 100644 --- a/wallet/test/run-db.c +++ b/wallet/test/run-db.c @@ -51,14 +51,6 @@ static struct db *create_test_db(void) return db; } -static int db_migration_count(void) -{ - int count = 0; - while (dbmigrations[count] != NULL) - count++; - return count - 1; -} - static bool test_empty_db_migrate(void) { struct db *db = create_test_db(); @@ -66,10 +58,9 @@ static bool test_empty_db_migrate(void) db_begin_transaction(db); CHECK(db_get_version(db) == -1); db_commit_transaction(db); - db_migrate(db, NULL); + db_migrate(NULL, db, NULL); db_begin_transaction(db); - CHECK(db_get_version(db) == db_migration_count()); - CHECK(db_get_version(db) == ARRAY_SIZE(dbmigrations) - 2); + CHECK(db_get_version(db) == ARRAY_SIZE(dbmigrations) - 1); db_commit_transaction(db); tal_free(db); @@ -105,7 +96,7 @@ static bool test_vars(void) struct db *db = create_test_db(); char *varname = "testvar"; CHECK(db); - db_migrate(db, NULL); + db_migrate(NULL, db, NULL); db_begin_transaction(db); /* Check default behavior */ diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 29ec197c1..8fc3cafea 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -671,7 +671,7 @@ static struct wallet *create_test_wallet(struct lightningd *ld, const tal_t *ctx w->bip32_base) == WALLY_OK); CHECK_MSG(w->db, "Failed opening the db"); - db_migrate(w->db, w->log); + db_migrate(NULL, w->db, w->log); CHECK_MSG(!wallet_err, "DB migration failed"); w->max_channel_dbid = 0; diff --git a/wallet/wallet.c b/wallet/wallet.c index 0e357ecd7..cee704dd5 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -50,7 +50,7 @@ struct wallet *wallet_new(struct lightningd *ld, { struct wallet *wallet = tal(ld, struct wallet); wallet->ld = ld; - wallet->db = db_setup(wallet, log); + wallet->db = db_setup(wallet, ld, log); wallet->log = log; wallet->bip32_base = NULL; list_head_init(&wallet->unstored_payments);