Migrated to postgres and added pub32 tests

This commit is contained in:
kexkey
2021-11-05 22:46:36 -04:00
parent da35fc50e3
commit 8a62c146a9
44 changed files with 2299 additions and 530 deletions

View File

@@ -22,7 +22,8 @@ createbatcher() {
local request=${1}
local response
local label=$(echo "${request}" | jq ".batcherLabel")
local returncode
local label=$(echo "${request}" | jq -r ".batcherLabel")
trace "[createbatcher] label=${label}"
local conf_target=$(echo "${request}" | jq ".confTarget")
trace "[createbatcher] conf_target=${conf_target}"
@@ -37,13 +38,18 @@ createbatcher() {
local batcher_id
batcher_id=$(sql "INSERT OR IGNORE INTO batcher (label, conf_target, feerate) VALUES (${label}, ${conf_target}, ${feerate}); SELECT LAST_INSERT_ROWID();")
batcher_id=$(sql "INSERT INTO batcher (label, conf_target, feerate)"\
" VALUES ('${label}', ${conf_target}, ${feerate})"\
" RETURNING id" \
"SELECT id FROM batcher WHERE label='${label}'")
returncode=$?
trace_rc ${returncode}
if ("${batcher_id}" -eq "0"); then
if ("${returncode}" -ne "0"); then
trace "[createbatcher] Could not insert"
response='{"result":null,"error":{"code":-32700,"message":"Could not create batcher, label probably already exists","data":'${request}'}}'
else
trace "[createbatcher] Inserted"
trace "[createbatcher] Inserted, response=${batcher_id}"
response='{"result":{"batcherId":'${batcher_id}'},"error":null}'
fi
@@ -79,7 +85,7 @@ updatebatcher() {
local id=$(echo "${request}" | jq ".batcherId")
trace "[updatebatcher] id=${id}"
local label=$(echo "${request}" | jq ".batcherLabel")
local label=$(echo "${request}" | jq -r ".batcherLabel")
trace "[updatebatcher] label=${label}"
local conf_target=$(echo "${request}" | jq ".confTarget")
trace "[updatebatcher] conf_target=${conf_target}"
@@ -99,12 +105,12 @@ updatebatcher() {
# fi
if [ "${id}" = "null" ]; then
whereclause="label=${label}"
whereclause="label='${label}'"
else
whereclause="id = ${id}"
fi
sql "UPDATE batcher set label=${label}, conf_target=${conf_target}, feerate=${feerate} WHERE ${whereclause}"
sql "UPDATE batcher set label='${label}', conf_target=${conf_target}, feerate=${feerate} WHERE ${whereclause}"
returncode=$?
trace_rc ${returncode}
if [ "${returncode}" -ne 0 ]; then
@@ -151,13 +157,13 @@ addtobatch() {
trace "[addtobatch] address=${address}"
local amount=$(echo "${request}" | jq ".amount")
trace "[addtobatch] amount=${amount}"
local label=$(echo "${request}" | jq ".outputLabel")
local label=$(echo "${request}" | jq -r ".outputLabel")
trace "[addtobatch] label=${label}"
local batcher_id=$(echo "${request}" | jq ".batcherId")
trace "[addtobatch] batcher_id=${batcher_id}"
local batcher_label=$(echo "${request}" | jq ".batcherLabel")
local batcher_label=$(echo "${request}" | jq -r ".batcherLabel")
trace "[addtobatch] batcher_label=${batcher_label}"
local webhook_url=$(echo "${request}" | jq ".webhookUrl")
local webhook_url=$(echo "${request}" | jq -r ".webhookUrl")
trace "[addtobatch] webhook_url=${webhook_url}"
# Let's lowercase bech32 addresses
@@ -185,7 +191,7 @@ addtobatch() {
if [ "${batcher_id}" = "null" ]; then
# Using batcher_label
batcher_id=$(sql "SELECT id FROM batcher WHERE label=${batcher_label}")
batcher_id=$(sql "SELECT id FROM batcher WHERE label='${batcher_label}'")
returncode=$?
trace_rc ${returncode}
fi
@@ -195,7 +201,7 @@ addtobatch() {
response='{"result":null,"error":{"code":-32700,"message":"batcher not found","data":'${request}'}}'
else
# Check if address already pending for this batcher...
inserted_id=$(sql "SELECT id FROM recipient WHERE LOWER(address)=LOWER(\"${address}\") AND tx_id IS NULL AND batcher_id=${batcher_id}")
inserted_id=$(sql "SELECT id FROM recipient WHERE LOWER(address)=LOWER('${address}') AND tx_id IS NULL AND batcher_id=${batcher_id}")
returncode=$?
trace_rc ${returncode}
@@ -211,7 +217,9 @@ addtobatch() {
fi
# Insert the new destination
inserted_id=$(sql "INSERT INTO recipient (address, amount, webhook_url, batcher_id, label) VALUES (\"${address}\", ${amount}, ${webhook_url}, ${batcher_id}, ${label}); SELECT LAST_INSERT_ROWID();")
inserted_id=$(sql "INSERT INTO recipient (address, amount, webhook_url, batcher_id, label)"\
" VALUES ('${address}', ${amount}, '${webhook_url}', ${batcher_id}, '${label}')"\
" RETURNING id")
returncode=$?
trace_rc ${returncode}
@@ -280,7 +288,7 @@ removefrombatch() {
if [ "${returncode}" -ne 0 ]; then
response='{"result":null,"error":{"code":-32700,"message":"Output was not removed","data":'${request}'}}'
else
row=$(sql "SELECT COUNT(id), COALESCE(MIN(inserted_ts), 0), COALESCE(SUM(amount), 0.00000000) FROM recipient WHERE tx_id IS NULL AND batcher_id=${batcher_id}")
row=$(sql "SELECT COUNT(id), COALESCE(MIN(inserted_ts), DATE '0001-01-01'), COALESCE(SUM(amount), 0.00000000) FROM recipient WHERE tx_id IS NULL AND batcher_id=${batcher_id}")
returncode=$?
trace_rc ${returncode}
@@ -336,7 +344,7 @@ batchspend() {
local batcher_id=$(echo "${request}" | jq ".batcherId")
trace "[batchspend] batcher_id=${batcher_id}"
local batcher_label=$(echo "${request}" | jq ".batcherLabel")
local batcher_label=$(echo "${request}" | jq -r ".batcherLabel")
trace "[batchspend] batcher_label=${batcher_label}"
local conf_target=$(echo "${request}" | jq ".confTarget")
trace "[batchspend] conf_target=${conf_target}"
@@ -351,7 +359,7 @@ batchspend() {
if [ "${batcher_id}" = "null" ]; then
# Using batcher_label
whereclause="label=${batcher_label}"
whereclause="label='${batcher_label}'"
else
whereclause="id=${batcher_id}"
fi
@@ -423,11 +431,11 @@ batchspend() {
trace "[batchspend] webhook_url=${webhook_url}"
if [ -z "${recipientsjson}" ]; then
whereclause="\"${recipient_id}\""
whereclause="${recipient_id}"
recipientsjson="\"${address}\":${amount}"
webhooks_data="{\"outputId\":${recipient_id},\"address\":\"${address}\",\"amount\":${amount},\"webhookUrl\":\"${webhook_url}\"}"
else
whereclause="${whereclause},\"${recipient_id}\""
whereclause="${whereclause},${recipient_id}"
recipientsjson="${recipientsjson},\"${address}\":${amount}"
webhooks_data="${webhooks_data},{\"outputId\":${recipient_id},\"address\":\"${address}\",\"amount\":${amount},\"webhookUrl\":\"${webhook_url}\"}"
fi
@@ -452,7 +460,7 @@ batchspend() {
tx_raw_details=$(get_rawtransaction ${txid} | tr -d '\n')
# Amounts and fees are negative when spending so we absolute those fields
local tx_hash=$(echo "${tx_raw_details}" | jq '.result.hash')
local tx_hash=$(echo "${tx_raw_details}" | jq -r '.result.hash')
local tx_ts_firstseen=$(echo "${tx_details}" | jq '.result.timereceived')
local tx_amount=$(echo "${tx_details}" | jq '.result.amount | fabs' | awk '{ printf "%.8f", $0 }')
local tx_size=$(echo "${tx_raw_details}" | jq '.result.size')
@@ -462,25 +470,20 @@ batchspend() {
tx_replaceable=$([ "${tx_replaceable}" = "yes" ] && echo "true" || echo "false")
trace "[batchspend] tx_replaceable=${tx_replaceable}"
local fees=$(echo "${tx_details}" | jq '.result.fee | fabs' | awk '{ printf "%.8f", $0 }')
# Sometimes raw tx are too long to be passed as paramater, so let's write
# it to a temp file for it to be read by sqlite3 and then delete the file
echo "${tx_raw_details}" > batchspend-rawtx-${txid}-$$.blob
# Get the info on the batch before setting it to done
row=$(sql "SELECT COUNT(id), COALESCE(MIN(inserted_ts), 0), COALESCE(SUM(amount), 0.00000000) FROM recipient WHERE tx_id IS NULL AND batcher_id=${batcher_id}")
row=$(sql "SELECT COUNT(id), COALESCE(MIN(inserted_ts), DATE '0001-01-01'), COALESCE(SUM(amount), 0.00000000) FROM recipient WHERE tx_id IS NULL AND batcher_id=${batcher_id}")
returncode=$?
trace_rc ${returncode}
# Let's insert the txid in our little DB -- then we'll already have it when receiving confirmation
sql_rawtx "INSERT OR IGNORE INTO rawtx (txid, hash, confirmations, timereceived, fee, size, vsize, is_replaceable, conf_target, raw_tx) VALUES (\"${txid}\", ${tx_hash}, 0, ${tx_ts_firstseen}, ${fees}, ${tx_size}, ${tx_vsize}, ${tx_replaceable}, ${conf_target}, readfile('batchspend-rawtx-${txid}-$$.blob'))"
trace_rc $?
id_inserted=$(sql "INSERT OR IGNORE INTO tx (txid, hash, confirmations, timereceived, fee, size, vsize, is_replaceable, conf_target) VALUES (\"${txid}\", ${tx_hash}, 0, ${tx_ts_firstseen}, ${fees}, ${tx_size}, ${tx_vsize}, ${tx_replaceable}, ${conf_target}); SELECT LAST_INSERT_ROWID();")
id_inserted=$(sql "INSERT INTO tx (txid, hash, confirmations, timereceived, fee, size, vsize, is_replaceable, conf_target)"\
" VALUES ('${txid}', '${tx_hash}', 0, ${tx_ts_firstseen}, ${fees}, ${tx_size}, ${tx_vsize}, ${tx_replaceable}, ${conf_target})"\
" RETURNING id" \
"SELECT id FROM tx WHERE txid='${txid}'")
returncode=$?
trace_rc ${returncode}
if [ "${returncode}" -eq 0 ]; then
if [ "${id_inserted}" -eq 0 ]; then
id_inserted=$(sql "SELECT id FROM tx WHERE txid=\"${txid}\"")
fi
trace "[batchspend] id_inserted: ${id_inserted}"
sql "UPDATE recipient SET tx_id=${id_inserted} WHERE id IN (${whereclause})"
trace_rc $?
@@ -495,13 +498,10 @@ batchspend() {
trace "[batchspend] total=${total}"
response='{"result":{"batcherId":'${batcher_id}',"confTarget":'${conf_target}',"nbOutputs":'${count}',"oldest":"'${oldest}'","total":'${total}
response="${response},\"status\":\"accepted\",\"txid\":\"${txid}\",\"hash\":${tx_hash},\"details\":{\"firstseen\":${tx_ts_firstseen},\"size\":${tx_size},\"vsize\":${tx_vsize},\"replaceable\":${tx_replaceable},\"fee\":${fees}},\"outputs\":[${webhooks_data}]}"
response="${response},\"status\":\"accepted\",\"txid\":\"${txid}\",\"hash\":\"${tx_hash}\",\"details\":{\"firstseen\":${tx_ts_firstseen},\"size\":${tx_size},\"vsize\":${tx_vsize},\"replaceable\":${tx_replaceable},\"fee\":${fees}},\"outputs\":[${webhooks_data}]}"
response="${response},\"error\":null}"
# Delete the temp file containing the raw tx (see above)
rm batchspend-rawtx-${txid}-$$.blob
batch_webhooks "[${webhooks_data}]" '"batcherId":'${batcher_id}',"confTarget":'${conf_target}',"nbOutputs":'${count}',"oldest":"'${oldest}'","total":'${total}',"status":"accepted","txid":"'${txid}'","hash":'${tx_hash}',"details":{"firstseen":'${tx_ts_firstseen}',"size":'${tx_size}',"vsize":'${tx_vsize}',"replaceable":'${tx_replaceable}',"fee":'${fees}'}'
batch_webhooks "[${webhooks_data}]" '"batcherId":'${batcher_id}',"confTarget":'${conf_target}',"nbOutputs":'${count}',"oldest":"'${oldest}'","total":'${total}',"status":"accepted","txid":"'${txid}'","hash":"'${tx_hash}'","details":{"firstseen":'${tx_ts_firstseen}',"size":'${tx_size}',"vsize":'${tx_vsize}',"replaceable":'${tx_replaceable}',"fee":'${fees}'}'
else
local message=$(echo "${data}" | jq -e ".error.message")
@@ -578,7 +578,7 @@ batch_check_webhooks() {
# I know this query for each output is not very efficient, but this function should not execute often, only in case of
# failed callbacks on batches...
# Get the info on the batch
row=$(sql "SELECT COUNT(id), COALESCE(MIN(inserted_ts), 0), COALESCE(SUM(amount), 0.00000000) FROM recipient r WHERE tx_id=\"${tx_id}\"")
row=$(sql "SELECT COUNT(id), COALESCE(MIN(inserted_ts), DATE '0001-01-01'), COALESCE(SUM(amount), 0.00000000) FROM recipient r WHERE tx_id='${tx_id}'")
# Use the selected row above
count=$(echo "${row}" | cut -d '|' -f1)
@@ -654,7 +654,7 @@ batch_webhooks() {
fi
done
sql "UPDATE recipient SET calledback=1, calledback_ts=CURRENT_TIMESTAMP WHERE id IN (${successful_recipient_ids})"
sql "UPDATE recipient SET calledback=true, calledback_ts=CURRENT_TIMESTAMP WHERE id IN (${successful_recipient_ids})"
trace_rc $?
}
@@ -671,7 +671,7 @@ listbatchers() {
# "error":null}
local batchers=$(sql "SELECT b.id, '{\"batcherId\":' || b.id || ',\"batcherLabel\":\"' || b.label || '\",\"confTarget\":' || conf_target || ',\"nbOutputs\":' || COUNT(r.id) || ',\"oldest\":\"' ||COALESCE(MIN(r.inserted_ts), 0) || '\",\"total\":' ||COALESCE(SUM(amount), 0.00000000) || '}' FROM batcher b LEFT JOIN recipient r ON r.batcher_id=b.id AND r.tx_id IS NULL GROUP BY b.id")
local batchers=$(sql "SELECT b.id, '{\"batcherId\":' || b.id || ',\"batcherLabel\":\"' || b.label || '\",\"confTarget\":' || conf_target || ',\"nbOutputs\":' || COUNT(r.id) || ',\"oldest\":\"' ||COALESCE(MIN(r.inserted_ts), DATE '0001-01-01') || '\",\"total\":' ||COALESCE(SUM(amount), 0.00000000) || '}' FROM batcher b LEFT JOIN recipient r ON r.batcher_id=b.id AND r.tx_id IS NULL GROUP BY b.id ORDER BY b.id")
trace "[listbatchers] batchers=${batchers}"
local returncode
@@ -717,7 +717,7 @@ getbatcher() {
local batcher_id=$(echo "${request}" | jq ".batcherId")
trace "[getbatcher] batcher_id=${batcher_id}"
local batcher_label=$(echo "${request}" | jq ".batcherLabel")
local batcher_label=$(echo "${request}" | jq -r ".batcherLabel")
trace "[getbatcher] batcher_label=${batcher_label}"
if [ "${batcher_id}" = "null" ] && [ "${batcher_label}" = "null" ]; then
@@ -728,13 +728,13 @@ getbatcher() {
if [ "${batcher_id}" = "null" ]; then
# Using batcher_label
whereclause="b.label=${batcher_label}"
whereclause="b.label='${batcher_label}'"
else
# Using batcher_id
whereclause="b.id=${batcher_id}"
fi
batcher=$(sql "SELECT b.id, '{\"batcherId\":' || b.id || ',\"batcherLabel\":\"' || b.label || '\",\"confTarget\":' || conf_target || ',\"nbOutputs\":' || COUNT(r.id) || ',\"oldest\":\"' ||COALESCE(MIN(r.inserted_ts), 0) || '\",\"total\":' ||COALESCE(SUM(amount), 0.00000000) || '}' FROM batcher b LEFT JOIN recipient r ON r.batcher_id=b.id AND r.tx_id IS NULL WHERE ${whereclause} GROUP BY b.id")
batcher=$(sql "SELECT b.id, '{\"batcherId\":' || b.id || ',\"batcherLabel\":\"' || b.label || '\",\"confTarget\":' || conf_target || ',\"nbOutputs\":' || COUNT(r.id) || ',\"oldest\":\"' ||COALESCE(MIN(r.inserted_ts), DATE '0001-01-01') || '\",\"total\":' ||COALESCE(SUM(amount), 0.00000000) || '}' FROM batcher b LEFT JOIN recipient r ON r.batcher_id=b.id AND r.tx_id IS NULL WHERE ${whereclause} GROUP BY b.id")
trace "[getbatcher] batcher=${batcher}"
if [ -n "${batcher}" ]; then
@@ -797,9 +797,9 @@ getbatchdetails() {
local batcher_id=$(echo "${request}" | jq ".batcherId")
trace "[getbatchdetails] batcher_id=${batcher_id}"
local batcher_label=$(echo "${request}" | jq ".batcherLabel")
local batcher_label=$(echo "${request}" | jq -r ".batcherLabel")
trace "[getbatchdetails] batcher_label=${batcher_label}"
local txid=$(echo "${request}" | jq ".txid")
local txid=$(echo "${request}" | jq -r ".txid")
trace "[getbatchdetails] txid=${txid}"
if [ "${batcher_id}" = "null" ] && [ "${batcher_label}" = "null" ]; then
@@ -810,7 +810,7 @@ getbatchdetails() {
if [ "${batcher_id}" = "null" ]; then
# Using batcher_label
whereclause="b.label=${batcher_label}"
whereclause="b.label='${batcher_label}'"
else
# Using batcher_id
whereclause="b.id=${batcher_id}"
@@ -818,7 +818,7 @@ getbatchdetails() {
if [ "${txid}" != "null" ]; then
# Using txid
whereclause="${whereclause} AND t.txid=${txid}"
whereclause="${whereclause} AND t.txid='${txid}'"
else
# null txid
whereclause="${whereclause} AND t.txid IS NULL"
@@ -826,7 +826,7 @@ getbatchdetails() {
fi
# First get the batch summary
batch=$(sql "SELECT b.id, COALESCE(t.id, NULL), '{\"batcherId\":' || b.id || ',\"batcherLabel\":\"' || b.label || '\",\"confTarget\":' || b.conf_target || ',\"nbOutputs\":' || COUNT(r.id) || ',\"oldest\":\"' ||COALESCE(MIN(r.inserted_ts), 0) || '\",\"total\":' ||COALESCE(SUM(amount), 0.00000000) FROM batcher b LEFT JOIN recipient r ON r.batcher_id=b.id ${outerclause} LEFT JOIN tx t ON t.id=r.tx_id WHERE ${whereclause} GROUP BY b.id")
batch=$(sql "SELECT b.id, COALESCE(t.id, NULL), '{\"batcherId\":' || b.id || ',\"batcherLabel\":\"' || b.label || '\",\"confTarget\":' || b.conf_target || ',\"nbOutputs\":' || COUNT(r.id) || ',\"oldest\":\"' ||COALESCE(MIN(r.inserted_ts), DATE '0001-01-01') || '\",\"total\":' ||COALESCE(SUM(amount), 0.00000000) FROM batcher b LEFT JOIN recipient r ON r.batcher_id=b.id ${outerclause} LEFT JOIN tx t ON t.id=r.tx_id WHERE ${whereclause} GROUP BY b.id, t.id")
trace "[getbatchdetails] batch=${batch}"
if [ -n "${batch}" ]; then
@@ -839,7 +839,7 @@ getbatchdetails() {
# Using txid
outerclause="AND r.tx_id=${tx_id}"
tx=$(sql "SELECT '\"txid\":\"' || txid || '\",\"hash\":\"' || hash || '\",\"details\":{\"firstseen\":' || timereceived || ',\"size\":' || size || ',\"vsize\":' || vsize || ',\"replaceable\":' || CASE is_replaceable WHEN 1 THEN 'true' ELSE 'false' END || ',\"fee\":' || fee || '}' FROM tx WHERE id=${tx_id}")
tx=$(sql "SELECT '\"txid\":\"' || txid || '\",\"hash\":\"' || hash || '\",\"details\":{\"firstseen\":' || timereceived || ',\"size\":' || size || ',\"vsize\":' || vsize || ',\"replaceable\":' || CASE is_replaceable WHEN true THEN 'true' ELSE 'false' END || ',\"fee\":' || fee || '}' FROM tx WHERE id=${tx_id}")
else
# null txid
outerclause="AND r.tx_id IS NULL"