pg as cyphernode, bug fixes and better tests

cyphernodeconf_docker/templates/installer/docker/docker-compose.yaml

@@ -8,6 +8,7 @@ services:
 
   postgres:
     image: postgres:<%= postgres_version %>
+    user: $USER
     environment:
       - "POSTGRES_USER=cyphernode"
       - "POSTGRES_PASSWORD=<%= postgres_password %>"

dist/setup.sh

@@ -388,7 +388,7 @@ install_docker() {
 
   if [ ! -d $POSTGRES_DATAPATH ]; then
     step "   create $POSTGRES_DATAPATH"
-    sudo_if_required mkdir -p $POSTGRES_DATAPATH
+    sudo_if_required mkdir -p $POSTGRES_DATAPATH/pgdata
     next
   fi
 
@@ -869,7 +869,7 @@ BITCOIN_VERSION="v22.0"
 LIGHTNING_VERSION="v0.10.2"
 TRAEFIK_VERSION="v1.7.9-alpine"
 MOSQUITTO_VERSION="1.6-openssl"
-POSTGRES_VERSION="14.0-alpine"
+POSTGRES_VERSION="14.0-bullseye"
 
 SETUP_DIR=$(dirname $(realpath $0))
 

proxy_docker/app/script/batching.sh

@@ -9,6 +9,8 @@ createbatcher() {
 
   # POST http://192.168.111.152:8080/createbatcher
   #
+  # Will UPDATE the batcher is it already exists (as per label)
+  #
   # args:
   # - batcherLabel, optional, id can be used to reference the batcher
   # - confTarget, optional, overriden by batchspend's confTarget, default Bitcoin Core conf_target will be used if not supplied
@@ -40,16 +42,18 @@ createbatcher() {
 
   batcher_id=$(sql "INSERT INTO batcher (label, conf_target, feerate)"\
 " VALUES ('${label}', ${conf_target}, ${feerate})"\
+" ON CONFLICT (label) DO"\
+" UPDATE SET conf_target=${conf_target}, feerate=${feerate}"\
 " RETURNING id" \
   "SELECT id FROM batcher WHERE label='${label}'")
   returncode=$?
   trace_rc ${returncode}
 
-  if ("${returncode}" -ne "0"); then
+  if [ "${returncode}" -ne "0" ]; then
     trace "[createbatcher] Could not insert"
-    response='{"result":null,"error":{"code":-32700,"message":"Could not create batcher, label probably already exists","data":'${request}'}}'
+    response='{"result":null,"error":{"code":-32700,"message":"Could not create/update batcher","data":'${request}'}}'
   else
-    trace "[createbatcher] Inserted, response=${batcher_id}"
+    trace "[createbatcher] Inserted or updated, response=${batcher_id}"
     response='{"result":{"batcherId":'${batcher_id}'},"error":null}'
   fi
 
@@ -653,8 +657,13 @@ batch_webhooks() {
     fi
   done
 
-  sql "UPDATE recipient SET calledback=true, calledback_ts=CURRENT_TIMESTAMP WHERE id IN (${successful_recipient_ids})"
-  trace_rc $?
+  if [ -n "${successful_recipient_ids}" ]; then
+    trace "[batch_webhooks] We have successful callbacks, let's update the db..."
+    sql "UPDATE recipient SET calledback=true, calledback_ts=CURRENT_TIMESTAMP WHERE id IN (${successful_recipient_ids})"
+    trace_rc $?
+  else
+    trace "[batch_webhooks] We don't have successful callbacks, no need to update the db!"
+  fi
 }
 
 listbatchers() {

proxy_docker/app/script/requesthandler.sh

@@ -494,6 +494,10 @@ main() {
           # BODY {}
           # BODY {"batcherId":34}
 
+          if [ "$http_method" = "GET" ]; then
+            line='{}'
+          fi
+
           response=$(getbatcher "${line}")
           returncode=$?
           ;;

proxy_docker/app/script/watchrequest.sh

@@ -276,6 +276,8 @@ watchpub32() {
   trace "[watchpub32] cb1conf_url=${cb1conf_url}, cb1conf_url_pg=${cb1conf_url_pg}"
 
   # upto_n is used when extending the watching window
+  # If this is supplied, it means we will not INSERT into watching_by_pub32, just add
+  # corresponding rows into watching
   local upto_n=${7}
   trace "[watchpub32] upto_n=${upto_n}"
 
@@ -289,68 +291,84 @@ watchpub32() {
   if [ -n "${upto_n}" ]; then
     # If upto_n provided, then we create from nstart to upto_n (instead of + GAP)
     last_n=${upto_n}
-  fi
-  local subspath=$(echo -e $path | sed -En "s/n/${nstart}-${last_n}/p")
-  trace "[watchpub32] subspath=${subspath}"
-  local addresses
-  addresses=$(derivepubpath '{"pub32":"'${pub32}'","path":"'${subspath}'"}')
-  returncode=$?
-  trace_rc ${returncode}
-#  trace "[watchpub32] addresses=${addresses}"
+  else
+    # If upto_n is not provided, it means it's a new watching_by_pub32 to insert,
+    # so let's make sure the label is not already in the table since label must
+    # be unique... but the key driver is pub32.
+    local row
+    row=$(sql "SELECT id, pub32, derivation_path, callback0conf, callback1conf, last_imported_n, watching, inserted_ts FROM watching_by_pub32 WHERE label=${label_pg}")
+    returncode=$?
+    trace_rc ${returncode}
 
-  if [ "${returncode}" -eq 0 ]; then
-#    result=$(create_wallet "${pub32}")
-#    returncode=$?
-#    trace_rc ${returncode}
-#    trace "[watchpub32request] result=${result}"
-    trace "[watchpub32] Skipping create_wallet"
+    if [ ${#row} -ne 0 ]; then
+      trace "[watchpub32] This label already exists in watching_by_pub32, must be unique."
+      error_msg="This label already exists in watching_by_pub32, must be unique."
+    fi
+  fi
+
+  if [ -z "${error_msg}" ]; then
+    local subspath=$(echo -e $path | sed -En "s/n/${nstart}-${last_n}/p")
+    trace "[watchpub32] subspath=${subspath}"
+    local addresses
+    addresses=$(derivepubpath '{"pub32":"'${pub32}'","path":"'${subspath}'"}')
+    returncode=$?
+    trace_rc ${returncode}
+  #  trace "[watchpub32] addresses=${addresses}"
 
     if [ "${returncode}" -eq 0 ]; then
-      # Importmulti in Bitcoin Core...
-      result=$(importmulti_rpc "${WATCHER_BTC_NODE_XPUB_WALLET}" "${pub32}" "${addresses}")
-      returncode=$?
-      trace_rc ${returncode}
-      trace "[watchpub32] result=${result}"
+  #    result=$(create_wallet "${pub32}")
+  #    returncode=$?
+  #    trace_rc ${returncode}
+  #    trace "[watchpub32request] result=${result}"
+      trace "[watchpub32] Skipping create_wallet"
 
       if [ "${returncode}" -eq 0 ]; then
-        if [ -n "${upto_n}" ]; then
-          # Update existing row, we are extending the watching window
-          id_inserted=$(sql "UPDATE watching_by_pub32 set last_imported_n=${upto_n} WHERE pub32=${pub32_pg} RETURNING id")
-          returncode=$?
-          trace_rc ${returncode}
-        else
-          # Insert in our DB...
-          id_inserted=$(sql "INSERT INTO watching_by_pub32 (pub32, label, derivation_path, watching, callback0conf, callback1conf, last_imported_n)"\
+        # Importmulti in Bitcoin Core...
+        result=$(importmulti_rpc "${WATCHER_BTC_NODE_XPUB_WALLET}" "${pub32}" "${addresses}")
+        returncode=$?
+        trace_rc ${returncode}
+        trace "[watchpub32] result=${result}"
+
+        if [ "${returncode}" -eq 0 ]; then
+          if [ -n "${upto_n}" ]; then
+            # Update existing row, we are extending the watching window
+            id_inserted=$(sql "UPDATE watching_by_pub32 set last_imported_n=${upto_n} WHERE pub32=${pub32_pg} RETURNING id")
+            returncode=$?
+            trace_rc ${returncode}
+          else
+            # Insert in our DB...
+            id_inserted=$(sql "INSERT INTO watching_by_pub32 (pub32, label, derivation_path, watching, callback0conf, callback1conf, last_imported_n)"\
 " VALUES (${pub32_pg}, ${label_pg}, ${path_pg}, true, ${cb0conf_url_pg}, ${cb1conf_url_pg}, ${last_n})"\
 " ON CONFLICT (pub32) DO"\
 " UPDATE SET watching=true, label=${label_pg}, callback0conf=${cb0conf_url_pg}, callback1conf=${cb1conf_url_pg}, derivation_path=${path_pg}, last_imported_n=${last_n}"\
 " RETURNING id" \
-          "SELECT id FROM watching_by_pub32 WHERE pub32=${pub32_pg}")
-          returncode=$?
-          trace_rc ${returncode}
-        fi
+            "SELECT id FROM watching_by_pub32 WHERE pub32=${pub32_pg}")
+            returncode=$?
+            trace_rc ${returncode}
+          fi
 
-        if [ "${returncode}" -eq 0 ]; then
-          trace "[watchpub32] id_inserted: ${id_inserted}"
+          if [ -n "${id_inserted}" ] && [ "${returncode}" -eq 0 ]; then
+            trace "[watchpub32] id_inserted: ${id_inserted}"
 
-          addresses=$(echo ${addresses} | jq -r ".addresses[].address")
-          insert_watches "${addresses}" "${label}" "${cb0conf_url}" "${cb1conf_url}" "${id_inserted}" "${nstart}"
-          returncode=$?
-          trace_rc ${returncode}
-          if [ "${returncode}" -ne 0 ]; then
-            error_msg="Can't insert xpub watches in DB"
+            addresses=$(echo ${addresses} | jq -r ".addresses[].address")
+            insert_watches "${addresses}" "${label}" "${cb0conf_url}" "${cb1conf_url}" "${id_inserted}" "${nstart}"
+            returncode=$?
+            trace_rc ${returncode}
+            if [ "${returncode}" -ne 0 ]; then
+              error_msg="Can't insert xpub watches in DB"
+            fi
+          else
+            error_msg="Can't insert xpub watcher in DB"
           fi
         else
-          error_msg="Can't insert xpub watcher in DB"
+          error_msg="Can't import addresses"
         fi
       else
-        error_msg="Can't import addresses"
+        error_msg="Can't create wallet"
       fi
     else
-      error_msg="Can't create wallet"
+      error_msg="Can't derive addresses"
     fi
-  else
-    error_msg="Can't derive addresses"
   fi
 
   if [ -z "${error_msg}" ]; then
@@ -367,10 +385,10 @@ watchpub32() {
   else
     data='{"error":"'${error_msg}'",'\
 '"event":"watchxpub",'\
-'"pub32":"'${pub32}'",'
+'"pub32":"'${pub32}'",'\
 '"label":"'${label}'",'\
 '"path":"'${path}'",'\
-'"nstart":${nstart},'\
+'"nstart":'${nstart}','\
 '"unconfirmedCallbackURL":'${cb0conf_url_json}','\
 '"confirmedCallbackURL":'${cb1conf_url_json}'}'
 

proxy_docker/app/tests/colors.sh

@@ -1,4 +1,3 @@
-#!/bin/sh
 
 # Reset
 Color_Off='\033[0m'       # Text Reset

proxy_docker/app/tests/mine.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # This needs to be run in regtest
 

proxy_docker/app/tests/startcallbackserver.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 date
 

proxy_docker/app/tests/test-batching.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 . ./colors.sh
 
@@ -42,7 +42,7 @@
 
 trace() {
   if [ "${1}" -le "${TRACING}" ]; then
-    echo "$(date -u +%FT%TZ) ${2}" 1>&2
+    echo -e "$(date -u +%FT%TZ) ${2}" 1>&2
   fi
 }
 
@@ -230,7 +230,7 @@ testbatching() {
   fi
 
   # List batchers (should show at least default and testbatcher batchers)
-  trace 2 "\n\n[testbatching] ${BCyan}Testing listbatches...${Color_Off}\n"
+  trace 2 "\n\n[testbatching] ${BCyan}Testing listbatchers...${Color_Off}\n"
   response=$(exec_in_test_container curl -s proxy:8888/listbatchers)
   trace 3 "[testbatching] response=${response}"
   id=$(echo "${response}" | jq '.result[] | select(.batcherLabel == "testbatcher") | .batcherId')

proxy_docker/app/tests/test-derive.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 . ./colors.sh
 
@@ -16,7 +16,7 @@
 
 trace() {
   if [ "${1}" -le "${TRACING}" ]; then
-    echo "$(date -u +%FT%TZ) ${2}" 1>&2
+    echo -e "$(date -u +%FT%TZ) ${2}" 1>&2
   fi
 }
 

proxy_docker/app/tests/test-manage-missed.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 . ./colors.sh
 . ./mine.sh
@@ -16,7 +16,7 @@
 
 trace() {
   if [ "${1}" -le "${TRACING}" ]; then
-    echo "$(date -u +%FT%TZ) ${2}" 1>&2
+    echo -e "$(date -u +%FT%TZ) ${2}" 1>&2
   fi
 }
 

proxy_docker/app/tests/test-watchpub32.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 . ./colors.sh
 . ./mine.sh
@@ -22,7 +22,7 @@
 
 trace() {
   if [ "${1}" -le "${TRACING}" ]; then
-    echo "$(date -u +%FT%TZ) ${2}" 1>&2
+    echo -e "$(date -u +%FT%TZ) ${2}" 1>&2
   fi
 }
 
@@ -113,9 +113,9 @@ test_watch_pub32() {
   trace 3 "[test_watch_pub32] data=${data}"
   response=$(exec_in_test_container curl -d "${data}" proxy:8888/watchxpub)
   trace 3 "[test_watch_pub32] response=${response}"
-  data=$(echo "${response}" | jq -re ".label")
-  if [ "${label1}" != "${data}" ]; then
-    trace 1 "\n\n[test_watch_pub32] ${On_Red}${BBlack} 1. watchxpub 1 failed!                                           ${Color_Off}\n"
+  data=$(echo "${response}" | jq -re ".error")
+  if [ "${?}" -eq "0" ]; then
+    trace 1 "\n\n[test_watch_pub32] ${On_Red}${BBlack} 1. watchxpub 1 failed: ${data}!                                           ${Color_Off}\n"
     return 10
   fi