A lot of small fixes, improvements on startup tests, migration

2025-12-17 04:35:14 +01:00 · 2021-11-27 00:31:18 -05:00
parent bfd9261890
commit ef4cd781da
17 changed files with 178 additions and 82 deletions
--- a/api_auth_docker/Dockerfile
+++ b/api_auth_docker/Dockerfile
@@ -1,7 +1,6 @@
 FROM nginx:1.18.0-alpine

 RUN apk add --update --no-cache \
-    bash \
    git \
    openssl \
    fcgiwrap \
--- a/api_auth_docker/entrypoint.sh
+++ b/api_auth_docker/entrypoint.sh
@@ -1,11 +1,11 @@
-#!/bin/bash
+#!/bin/sh

 while [ ! -f "/container_monitor/proxy_ready" ]; do echo "proxy not ready" ; sleep 10 ; done

 echo "proxy ready"

-if [[ $1 ]]; then
-	user=$(echo $1 | cut -d ':' -f 1)
+if [ -n "$1" ]; then
+	user=$(echo "$1" | cut -d ':' -f 1)
 else
 	user='nginx'
 fi
--- a/cyphernodeconf_docker/templates/installer/docker/docker-compose.yaml
+++ b/cyphernodeconf_docker/templates/installer/docker/docker-compose.yaml
@@ -9,6 +9,7 @@ services:
  postgres:
    image: postgres:<%= postgres_version %>
    user: $USER
+    command: -c logging_collector=true -c log_directory=/cnlogs/
    environment:
      - "POSTGRES_USER=cyphernode"
      - "POSTGRES_PASSWORD=<%= postgres_password %>"
@@ -16,6 +17,7 @@ services:
      - "PGDATA=/var/lib/postgresql/data/pgdata"
    volumes:
      - "<%= postgres_datapath %>:/var/lib/postgresql/data"
+      - "<%= logs_datapath %>:/cnlogs"
    networks:
      - cyphernodenet
    <% if ( docker_mode === 'swarm' ) { %>
--- a/cyphernodeconf_docker/templates/installer/testdeployment.sh
+++ b/cyphernodeconf_docker/templates/installer/testdeployment.sh
@@ -65,14 +65,21 @@ if [ -f $current_path/exitStatus.sh ]; then
  rm -f $current_path/exitStatus.sh
 fi

-test_apps
+if [ "$EXIT_STATUS" -ne "0" ]; then
+  printf "\r\n\033[1;31mSkipping cypherapps deployment because of previous errors.\r\n\r\n\033[0m"
+else
+  test_apps
+fi

 EXIT_STATUS=$(($? | ${EXIT_STATUS}))

 printf "\r\n\e[1;32mTests finished.\e[0m\n"

 if [ "$EXIT_STATUS" -ne "0" ]; then
-  printf "\r\n\033[1;31mThere was an error during cyphernode installation.  full logs:  docker ps -q | xargs -L 1 docker logs , Containers logs:  docker logs <containerid> , list containers: docker ps  .Please see Docker's logs for more information.  Run ./testdeployment.sh to rerun the tests.  Run ./stop.sh to stop cyphernode.\r\n\r\n\033[0m"
+  printf "\r\n\033[1;31mThere was an error during cyphernode installation.\r\n\033[0m"
+  printf "\r\n\033[1;31mCheck logs in your logs directory (${LOGS_DATAPATH}).\r\n\033[0m"
+  printf "\r\n\033[1;31mRun ./testdeployment.sh to rerun the tests.\033[0m"
+  printf "\r\n\033[1;31mRun ./stop.sh to stop cyphernode.\r\n\033[0m"
  exit 1
 fi

--- a/cyphernodeconf_docker/templates/installer/testfeatures.sh
+++ b/cyphernodeconf_docker/templates/installer/testfeatures.sh
@@ -1,6 +1,6 @@
 #!/bin/sh

-apk add --update --no-cache openssl curl jq coreutils > /dev/null
+apk add --update --no-cache openssl curl jq coreutils postgresql > /dev/null

 . /gatekeeper/keys.properties

@@ -72,7 +72,7 @@ checkpycoin() {
  echo -en "\r\n\e[1;36mTesting Pycoin... " > /dev/console
  local rc

-  rc=$(curl -H "Content-Type: application/json" -d "{\"pub32\":\"upub5GtUcgGed1aGH4HKQ3vMYrsmLXwmHhS1AeX33ZvDgZiyvkGhNTvGd2TA5Lr4v239Fzjj4ZY48t6wTtXUy2yRgapf37QHgt6KWEZ6bgsCLpb\",\"path\":\"0/25-30\"}" -s -o /dev/null -w "%{http_code}" http://proxy:8888/derivepubpath)
+  rc=$(curl -H "Content-Type: application/json" -d "{\"pub32\":\"upub5GtUcgGed1aGH4HKQ3vMYrsmLXwmHhS1AeX33ZvDgZiyvkGhNTvGd2TA5Lr4v239Fzjj4ZY48t6wTtXUy2yRgapf37QHgt6KWEZ6bgsCLpb\",\"path\":\"0/25-30\"}" -s -o /dev/null -w "%{http_code}" http://pycoin:7777/derive)
  [ "${rc}" -ne "200" ] && return 100

  echo -e "\e[1;36mPycoin rocks!" > /dev/console
@@ -85,8 +85,9 @@ checkpostgres() {
  local rc

  # getbatcher needs the database to return correctly...
-  rc=$(curl -s -o /dev/null -w "%{http_code}" http://proxy:8888/getbatcher)
-  [ "${rc}" -ne "200" ] && return 105
+#  rc=$(curl -s -o /dev/null -w "%{http_code}" http://proxy:8888/getbatcher)
+  pg_isready -h postgres
+  [ "${?}" -ne "0" ] && return 105

  echo -e "\e[1;36mPostgres rocks!" > /dev/console

@@ -110,7 +111,9 @@ checknotifier() {
  local response
  local returncode

-  response=$(mosquitto_rr -h broker -W 15 -t notifier -e "response/$$" -m "{\"response-topic\":\"response/$$\",\"cmd\":\"web\",\"url\":\"http://proxy:8888/helloworld\",\"tor\":false}")
+  nc -vlp1111 -e sh -c 'echo -en "HTTP/1.1 200 OK\\r\\n\\r\\n" ; date >&2 ; timeout 1 tee /dev/tty | cat ; ' &
+  # response=$(mosquitto_rr -h broker -W 15 -t notifier -e "response/$$" -m "{\"response-topic\":\"response/$$\",\"cmd\":\"web\",\"url\":\"http://proxy:8888/helloworld\",\"tor\":false}")
+  response=$(mosquitto_rr -h broker -W 15 -t notifier -e "response/$$" -m "{\"response-topic\":\"response/$$\",\"cmd\":\"web\",\"url\":\"http://$(hostname):1111/notifiertest\",\"tor\":false}")
  returncode=$?
  [ "${returncode}" -ne "0" ] && return 115
  http_code=$(echo "${response}" | jq -r ".http_code")
@@ -125,7 +128,8 @@ checkots() {
  echo -en "\r\n\e[1;36mTesting OTSclient... " > /dev/console
  local rc

-  rc=$(curl -s -H "Content-Type: application/json" -d '{"hash":"123","callbackUrl":"http://callback"}' http://proxy:8888/ots_stamp)
+  # rc=$(curl -s -H "Content-Type: application/json" -d '{"hash":"123","callbackUrl":"http://callback"}' http://proxy:8888/ots_stamp)
+  rc=$(curl -s otsclient:6666/stamp/123)
  echo "${rc}" | grep "Invalid hash 123 for sha256" > /dev/null
  [ "$?" -ne "0" ] && return 200

@@ -301,7 +305,7 @@ if [ "${returncode}" -ne "0" ]; then
  echo -e "\e[1;31mCyphernode could not fully start properly within delay." > /dev/console
  status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"proxy\") | .active")
  if [ "${status}" = "false" ]; then
-    echo -e "\e[1;31mThe Proxy, the main Cyphernode's component, is not responding.  We will only test the gatekeeper if its container is up, but you'll see errors for the other components.  Please check the logs." > /dev/console
+    echo -e "\r\n\e[1;31mThe Proxy, the main Cyphernode's component, is not responding.  You'll see errors for the other components.  Please check the logs." > /dev/console
    workingproxy="false"
  fi
 else
@@ -323,14 +327,32 @@ fi
 # PROXY                     #
 #############################

-echo -e "\r\n\e[1;36mWaiting for Proxy to be ready... " > /dev/console
-timeout_feature '[ -f "/container_monitor/proxy_ready" ]' 300
+if [ ! -f /container_monitor/proxy_dbfailed ]; then
+  echo -e "\r\n\e[1;36mWaiting for Proxy to be ready... " > /dev/console
+  timeout_feature '[ -f "/container_monitor/proxy_ready" ]' 300
+  returncode=$?
+  if [ "${returncode}" -ne "0" ]; then
+    echo -e "\r\n\e[1;31mThe proxy is still not ready.  It may be migrating large quantity of data?  Please check the logs for more details." > /dev/console
+    workingproxy="false"
+  fi
+fi
+if [ -f /container_monitor/proxy_dbfailed ]; then
+  echo -e "\r\n\e[1;31mThe proxy's database migration failed.  Please check proxy.log for more details." > /dev/console
+  workingproxy="false"
+fi
+
+if [ "${workingproxy}" = "false" ]; then
+  echo -e "\r\n\e[1;31mThe Proxy, the main Cyphernode's component, is not ready.  Cyphernode can't be run without the proxy component." > /dev/console
+  echo -e "\r\n\e[1;31mThe other components will fail next, this is normal." > /dev/console
+fi
+
+result="${containers},\"features\":[{\"coreFeature\":true,\"name\":\"proxy\",\"working\":${workingproxy}}"

 #############################
 # POSTGRES                  #
 #############################

-result="${result},{\"coreFeature\":true, \"name\":\"postgres\",\"working\":"
+result="${result},{\"coreFeature\":true,\"name\":\"postgres\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"postgres\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
  timeout_feature checkpostgres
@@ -345,9 +367,9 @@ result="${result}$(feature_status ${returncode} 'Postgres error!')}"
 # GATEKEEPER                #
 #############################

-result="${containers},\"features\":[{\"coreFeature\":true, \"name\":\"proxy\",\"working\":${workingproxy}}, {\"coreFeature\":true, \"name\":\"gatekeeper\",\"working\":"
+result="${result},{\"coreFeature\":true,\"name\":\"gatekeeper\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"gatekeeper\") | .active")
-if [ "${status}" = "true" ]; then
+if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
  timeout_feature checkgatekeeper
  returncode=$?
 else
@@ -360,7 +382,7 @@ result="${result}$(feature_status ${returncode} 'Gatekeeper error!')}"
 # BROKER                    #
 #############################

-result="${result},{\"coreFeature\":true, \"name\":\"broker\",\"working\":"
+result="${result},{\"coreFeature\":true,\"name\":\"broker\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"broker\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
  timeout_feature checkbroker
@@ -375,7 +397,7 @@ result="${result}$(feature_status ${returncode} 'Broker error!')}"
 # NOTIFIER                  #
 #############################

-result="${result},{\"coreFeature\":true, \"name\":\"notifier\",\"working\":"
+result="${result},{\"coreFeature\":true,\"name\":\"notifier\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"notifier\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
  timeout_feature checknotifier
@@ -390,7 +412,7 @@ result="${result}$(feature_status ${returncode} 'Notifier error!')}"
 # PYCOIN                    #
 #############################

-result="${result},{\"coreFeature\":true, \"name\":\"pycoin\",\"working\":"
+result="${result},{\"coreFeature\":true,\"name\":\"pycoin\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"pycoin\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
  timeout_feature checkpycoin
@@ -406,7 +428,7 @@ result="${result}$(feature_status ${returncode} 'Pycoin error!')}"
 # OTSCLIENT                 #
 #############################

-result="${result},{\"coreFeature\":false, \"name\":\"otsclient\",\"working\":"
+result="${result},{\"coreFeature\":false,\"name\":\"otsclient\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"otsclient\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
  timeout_feature checkots
@@ -423,12 +445,11 @@ result="${result}$(feature_status ${returncode} 'OTSclient error!')}"
 # TOR                       #
 #############################

-echo -e "\r\n\e[1;36mWaiting for Tor to be ready... " > /dev/console
-timeout_feature '[ -f "/container_monitor/tor_ready" ]'
-
-result="${result},{\"coreFeature\":false, \"name\":\"tor\",\"working\":"
+result="${result},{\"coreFeature\":false,\"name\":\"tor\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"tor\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
+  echo -e "\r\n\e[1;36mWaiting for Tor to be ready... " > /dev/console
+  timeout_feature '[ -f "/container_monitor/tor_ready" ]'
  timeout_feature checktor
  returncode=$?
 else
@@ -442,12 +463,11 @@ result="${result}$(feature_status ${returncode} 'Tor error!')}"
 # BITCOIN                   #
 #############################

-echo -e "\r\n\e[1;36mWaiting for Bitcoin Core to be ready... " > /dev/console
-timeout_feature '[ -f "/container_monitor/bitcoin_ready" ]'
-
-result="${result},{\"coreFeature\":true, \"name\":\"bitcoin\",\"working\":"
+result="${result},{\"coreFeature\":true,\"name\":\"bitcoin\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"bitcoin\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
+  echo -e "\r\n\e[1;36mWaiting for Bitcoin Core to be ready... " > /dev/console
+  timeout_feature '[ -f "/container_monitor/bitcoin_ready" ]'
  timeout_feature checkbitcoinnode
  returncode=$?
 else
@@ -461,12 +481,11 @@ result="${result}$(feature_status ${returncode} 'Bitcoin error!')}"
 # LIGHTNING                 #
 #############################

-echo -e "\r\n\e[1;36mWaiting for C-Lightning to be ready... " > /dev/console
-timeout_feature '[ -f "/container_monitor/lightning_ready" ]'
-
-result="${result},{\"coreFeature\":false, \"name\":\"lightning\",\"working\":"
+result="${result},{\"coreFeature\":false,\"name\":\"lightning\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"lightning\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
+  echo -e "\r\n\e[1;36mWaiting for C-Lightning to be ready... " > /dev/console
+  timeout_feature '[ -f "/container_monitor/lightning_ready" ]'
  timeout_feature checklnnode
  returncode=$?
 else
--- a/dist/setup.sh
+++ b/dist/setup.sh
@@ -16,6 +16,20 @@
 #
 # Let's say if timer > 2, we're on a slow machine.

+# At first we tried using uname -m te determine slow devices, but:
+#
+# uname -m result:
+# RPi2: armv7l
+# RPi3: armv7l
+# RPi4 on 32-bit OS: armv7l
+# RPi4 on 64-bit OS: aarch64
+# RockPi: aarch64
+# Apple M1: arm64
+# Intel 64: x86_64#
+#
+# There are a ton of other possible values... and can't rely on them to detect
+# a slow device.
+
 # FROM: https://stackoverflow.com/questions/5195607/checking-bash-exit-status-of-several-commands-efficiently
 # Use step(), try(), and next() to perform a series of commands and print
 # [  OK  ] or [FAILED] at the end. The step as a whole fails if any individual
--- a/notifier_docker/script/requesthandler.sh
+++ b/notifier_docker/script/requesthandler.sh
@@ -35,5 +35,6 @@ main() {
 }

 main
+returncode=$?
 trace "[requesthandler] exiting"
-exit $?
+exit ${returncode}
--- a/otsclient_docker/script/requesthandler.sh
+++ b/otsclient_docker/script/requesthandler.sh
@@ -97,8 +97,10 @@ main()
    fi
  done
  trace "[main] exiting"
-  return 0
+  return ${returncode}
 }

 main
-exit $?
+returncode=$?
+trace "[requesthandler] exiting"
+exit ${returncode}
--- a/proxy_docker/app/data/cyphernode.postgresql
+++ b/proxy_docker/app/data/cyphernode.postgresql
@@ -136,8 +136,8 @@ CREATE TABLE cyphernode_props (
 CREATE INDEX idx_cp_property ON cyphernode_props (property);
 CREATE UNIQUE INDEX idx_cp_propval ON cyphernode_props (property, value);

-INSERT INTO cyphernode_props (property, value) VALUES ('version', '0.1');
-INSERT INTO cyphernode_props (property, value) VALUES ('pay_index', '0');
+INSERT INTO cyphernode_props (id, property, value) VALUES (1, 'version', '0.1');
+INSERT INTO cyphernode_props (id, property, value) VALUES (2, 'pay_index', '0');

 CREATE TABLE ln_invoice (
  id SERIAL PRIMARY KEY,
--- a/proxy_docker/app/data/sqlmigrate20210808_0.7.0-0.8.0.sh
+++ b/proxy_docker/app/data/sqlmigrate20210808_0.7.0-0.8.0.sh
@@ -1,14 +1,19 @@
 #!/bin/sh

-echo "Checking for labels for watched addresses support in DB..."
+. ./trace.sh
+
+trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Checking for labels for watched addresses support in DB..."
 count=$(sqlite3 $DB_FILE "select count(*) from pragma_table_info('watching') where name='label'")
 if [ "${count}" -eq "0" ]; then
-	# label not there, we have to migrate
-	echo "Migrating database for labels for watched addresses support..."
-  echo "Backing up current DB..."
+  # label not there, we have to migrate
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Migrating database for labels for watched addresses support..."
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Backing up current DB..."
  cp  $DB_FILE $DB_FILE-sqlmigrate20210808_0.7.0-0.8.0
-  echo "Altering DB..."
-	cat sqlmigrate20210808_0.7.0-0.8.0.sql | sqlite3 $DB_FILE
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Altering DB..."
+  cat sqlmigrate20210808_0.7.0-0.8.0.sql | sqlite3 $DB_FILE
+  returncode=$?
+  trace_rc ${returncode}
+  exit ${returncode}
 else
-	echo "Database labels for watched addresses support migration already done, skipping!"
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Database labels for watched addresses support migration already done, skipping!"
 fi
--- a/proxy_docker/app/data/sqlmigrate20210928_0.7.0-0.8.0.sh
+++ b/proxy_docker/app/data/sqlmigrate20210928_0.7.0-0.8.0.sh
@@ -1,14 +1,19 @@
 #!/bin/sh

-echo "Checking for new indexes in DB..."
+. ./trace.sh
+
+trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Checking for new indexes in DB..."
 sqlite3 $DB_FILE ".indexes" | grep "idx_watching_watching" > /dev/null
 if [ "$?" -eq "1" ]; then
  # idx_watching_watching index not found
-  echo "Migrating database with new indexes..."
-  echo "Backing up current DB..."
-  cp  $DB_FILE $DB_FILE-sqlmigrate20210928_0.7.0-0.8.0
-  echo "Altering DB..."
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Migrating database with new indexes..."
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Backing up current DB..."
+  cp $DB_FILE $DB_FILE-sqlmigrate20210928_0.7.0-0.8.0
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Altering DB..."
  cat sqlmigrate20210928_0.7.0-0.8.0.sql | sqlite3 $DB_FILE
+  returncode=$?
+  trace_rc ${returncode}
+  exit ${returncode}
 else
-  echo "New indexes migration already done, skipping!"
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] New indexes migration already done, skipping!"
 fi
--- a/proxy_docker/app/data/sqlmigrate20211105_0.7.0-0.8.0.sh
+++ b/proxy_docker/app/data/sqlmigrate20211105_0.7.0-0.8.0.sh
@@ -1,26 +1,39 @@
 #!/bin/sh

-date
-echo "Waiting for postgres to be ready..."
-(while true ; do psql -h postgres -U cyphernode -c "select 1;" ; [ "$?" -eq "0" ] && break ; sleep 10; done) &
-wait
+. ./trace.sh

-echo "Checking if postgres is setup..."
+trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Waiting for postgres to be ready..."
+while true ; do psql -h postgres -U cyphernode -c "select 1;" ; [ "$?" -eq "0" ] && break ; sleep 10; done
+
+trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Checking if postgres is setup..."
 psql -h postgres -U cyphernode -c "\d" | grep "cyphernode_props" > /dev/null
 if [ "$?" -eq "1" ]; then
  # if cyphernode_props table doesn't exist, it's probably because database hasn't been setup yet
-  echo "Creating postgres database..."
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Creating postgres database..."
  psql -h postgres -f cyphernode.postgresql -U cyphernode
+  returncode=$?
+  trace_rc ${returncode}
+  [ "${returncode}" -eq "0" ] || exit ${returncode}
+else
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] PostgreSQL database already created, skipping!"
+fi

-  date
-  echo "Extracting and converting sqlite3 data..."
-  date
+trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Checking if postgres is loaded/imported..."
+lastval=$(psql -qAtX -h postgres -U cyphernode -c "select last_value from pg_sequences where sequencename='cyphernode_props_id_seq'")
+returncode=$?
+if [ -z "${lastval}" ] || [ "${lastval}" -lt "2" ]; then
+  # if cyphernode_props_id_seq isn't set, it's probably because database hasn't been loaded/imported yet
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Extracting and converting sqlite3 data..."
  cat sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extract.sql | sqlite3 $DB_FILE
+  returncode=$?
+  trace_rc ${returncode}
+  [ "${returncode}" -eq "0" ] || exit ${returncode}
+
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Creating import file for postgres..."
  mv sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql ${DB_PATH}/
  sed -ie 's/^\(INSERT.*\);$/\1 ON CONFLICT DO NOTHING;/g' ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql

-  date
-  echo "...appending postgresql sequences..."
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Appending postgresql sequence creation..."
  echo "
 select setval('cyphernode_props_id_seq',  (SELECT MAX(id) FROM cyphernode_props));
 select setval('ln_invoice_id_seq',  (SELECT MAX(id) FROM ln_invoice));
@@ -34,10 +47,11 @@ select setval('batcher_id_seq',  (SELECT MAX(id) FROM batcher));
 commit;
 " >> ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql

-  date
-  echo "Importing sqlite3 data into postgresql..."
-  psql -h postgres -f ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql -U cyphernode
-  date
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Importing sqlite3 data into postgresql..."
+  psql -v ON_ERROR_STOP=on -h postgres -f ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql -U cyphernode
+  returncode=$?
+  trace_rc ${returncode}
+  [ "${returncode}" -eq "0" ] || exit ${returncode}
 else
-  echo "New indexes migration already done, skipping!"
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] PostgreSQL database already loaded, skipping!"
 fi
--- a/proxy_docker/app/data/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extract.sql
+++ b/proxy_docker/app/data/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extract.sql
@@ -17,11 +17,10 @@ select id,address,amount,tx_id,inserted_ts,webhook_url,case when calledback=1 th
 select id,txid,case when watching=1 then 'TRUE' else 'FALSE' end as watching,callback1conf,case when calledback1conf=1 then 'TRUE' else 'FALSE' end as calledback1conf,callbackxconf,case when calledbackxconf=1 then 'TRUE' else 'FALSE' end as calledbackxconf,nbxconf,inserted_ts from watching_by_txid;
 .mode insert stamp
 select id,hash,callbackUrl,case when requested=1 then 'TRUE' else 'FALSE' end as requested,case when upgraded=1 then 'TRUE' else 'FALSE' end as upgraded,case when calledback=1 then 'TRUE' else 'FALSE' end as calledback,inserted_ts from stamp;
+.mode insert ln_invoice
+select id,label,bolt11,payment_hash,msatoshi,status,pay_index,msatoshi_received,paid_at,description,expires_at,callback_url,case when calledback=1 then 'TRUE' else 'FALSE' end as calledback,case when callback_failed=1 then 'TRUE' else 'FALSE' end as callback_failed,inserted_ts from ln_invoice;
 -- cyphernode_props rows were already inserted in db creation, let's update them here
 .headers off
 .mode list cyphernode_props
 select 'update cyphernode_props set value=''' || value || ''', inserted_ts=''' || inserted_ts || ''' where id=' || id || ';' from cyphernode_props;
-.headers on
-.mode insert ln_invoice
-select id,label,bolt11,payment_hash,msatoshi,status,pay_index,msatoshi_received,paid_at,description,expires_at,callback_url,case when calledback=1 then 'TRUE' else 'FALSE' end as calledback,case when callback_failed=1 then 'TRUE' else 'FALSE' end as callback_failed,inserted_ts from ln_invoice;
 .quit
--- a/proxy_docker/app/script/computefees.sh
+++ b/proxy_docker/app/script/computefees.sh
@@ -11,7 +11,7 @@ compute_fees() {
    trace "[compute_fees]  pruned=${pruned}"
    # We want null instead of 0.00000000 in this case.
    echo "null"
-    exit 0
+    return
  fi

  local txid=${1}
--- a/proxy_docker/app/script/requesthandler.sh
+++ b/proxy_docker/app/script/requesthandler.sh
@@ -771,9 +771,10 @@ main() {
    fi
  done
  trace "[main] exiting"
-  return 0
+  return ${returncode}
 }

 main
+returncode=$?
 trace "[requesthandler] exiting"
-exit $?
+exit ${returncode}
--- a/proxy_docker/app/script/startproxy.sh
+++ b/proxy_docker/app/script/startproxy.sh
@@ -1,5 +1,7 @@
 #!/bin/sh

+. ./trace.sh
+
 trim() {
  echo -e "$1" | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//'
 }
@@ -7,12 +9,12 @@ trim() {
 createCurlConfig() {

  if [[ ''$1 == '' ]]; then
-    echo "Missing file name: Check your *_BTC_NODE_RPC_CFG"
+    trace "[startproxy] Missing file name: Check your *_BTC_NODE_RPC_CFG"
    return
  fi

  if [[ ''$2 == '' ]]; then
-    echo "Missing content: Check your *_BTC_NODE_RPC_USER"
+    trace "[startproxy] Missing content: Check your *_BTC_NODE_RPC_USER"
    return
  fi

@@ -21,17 +23,45 @@ createCurlConfig() {

 }

+if [ -e ${DB_PATH}/.dbfailed ]; then
+  touch /container_monitor/proxy_dbfailed
+  trace "[startproxy] A previous database creation/migration failed.  Stopping."
+  trace "[startproxy] A file called .dbfailed has been created.  Fix the migration errors, remove .dbfailed and retry."
+  trace "[startproxy] Exiting."
+  sleep 30
+  exit 1
+else
+  rm -f /container_monitor/proxy_dbfailed
+fi
+
 if [ ! -e ${DB_FILE} ]; then
-  echo "DB not found, creating..."
+  trace "[startproxy] DB not found, creating..."
  cat cyphernode.sql | sqlite3 $DB_FILE
  psql -h postgres -f cyphernode.postgresql -U cyphernode
+  returncode=$?
+  trace_rc ${returncode}
 else
-  echo "DB found, migrating..."
+  trace "[startproxy] DB found, migrating..."
  for script in sqlmigrate*.sh; do
    sh $script
+    returncode=$?
+    trace_rc ${returncode}
+    if [ "${returncode}" -ne "0" ]; then
+      break
+    fi
  done
 fi

+if [ "${returncode}" -ne "0" ]; then
+  touch ${DB_PATH}/.dbfailed
+  touch /container_monitor/proxy_dbfailed
+  trace "[startproxy] Database creation/migration failed.  Stopping."
+  trace "[startproxy] A file called .dbfailed has been created in your proxy datapath.  Fix the migration errors, remove .dbfailed and retry."
+  trace "[startproxy] Exiting."
+  sleep 30
+  exit ${returncode}
+fi
+
 chmod 0600 $DB_FILE

 createCurlConfig ${WATCHER_BTC_NODE_RPC_CFG} ${WATCHER_BTC_NODE_RPC_USER}
--- a/proxy_docker/app/script/trace.sh
+++ b/proxy_docker/app/script/trace.sh
@@ -1,7 +1,6 @@
 #!/bin/sh

-trace()
-{
+trace() {
  if [ -n "${TRACING}" ]; then
    local str="$(date -Is) $$ ${1}"
    echo "${str}" 1>&2
@@ -9,8 +8,7 @@ trace()
  fi
 }

-trace_rc()
-{
+trace_rc() {
  if [ -n "${TRACING}" ]; then
    local str="$(date -Is) $$ Last return code: ${1}"
    echo "${str}" 1>&2