A lot of small fixes, improvements on startup tests, migration

api_auth_docker/Dockerfile

@@ -1,7 +1,6 @@
 FROM nginx:1.18.0-alpine
 
 RUN apk add --update --no-cache \
-    bash \
     git \
     openssl \
     fcgiwrap \

api_auth_docker/entrypoint.sh

@@ -1,11 +1,11 @@
-#!/bin/bash
+#!/bin/sh
 
 while [ ! -f "/container_monitor/proxy_ready" ]; do echo "proxy not ready" ; sleep 10 ; done
 
 echo "proxy ready"
 
-if [[ $1 ]]; then
+if [ -n "$1" ]; then
-	user=$(echo $1 | cut -d ':' -f 1)
+	user=$(echo "$1" | cut -d ':' -f 1)
 else
 	user='nginx'
 fi

cyphernodeconf_docker/templates/installer/docker/docker-compose.yaml

@@ -9,6 +9,7 @@ services:
   postgres:
     image: postgres:<%= postgres_version %>
     user: $USER
+    command: -c logging_collector=true -c log_directory=/cnlogs/
     environment:
       - "POSTGRES_USER=cyphernode"
       - "POSTGRES_PASSWORD=<%= postgres_password %>"
@@ -16,6 +17,7 @@ services:
       - "PGDATA=/var/lib/postgresql/data/pgdata"
     volumes:
       - "<%= postgres_datapath %>:/var/lib/postgresql/data"
+      - "<%= logs_datapath %>:/cnlogs"
     networks:
       - cyphernodenet
     <% if ( docker_mode === 'swarm' ) { %>

cyphernodeconf_docker/templates/installer/testdeployment.sh

@@ -65,14 +65,21 @@ if [ -f $current_path/exitStatus.sh ]; then
   rm -f $current_path/exitStatus.sh
 fi
 
+if [ "$EXIT_STATUS" -ne "0" ]; then
+  printf "\r\n\033[1;31mSkipping cypherapps deployment because of previous errors.\r\n\r\n\033[0m"
+else
   test_apps
+fi
 
 EXIT_STATUS=$(($? | ${EXIT_STATUS}))
 
 printf "\r\n\e[1;32mTests finished.\e[0m\n"
 
 if [ "$EXIT_STATUS" -ne "0" ]; then
-  printf "\r\n\033[1;31mThere was an error during cyphernode installation.  full logs:  docker ps -q | xargs -L 1 docker logs , Containers logs:  docker logs <containerid> , list containers: docker ps  .Please see Docker's logs for more information.  Run ./testdeployment.sh to rerun the tests.  Run ./stop.sh to stop cyphernode.\r\n\r\n\033[0m"
+  printf "\r\n\033[1;31mThere was an error during cyphernode installation.\r\n\033[0m"
+  printf "\r\n\033[1;31mCheck logs in your logs directory (${LOGS_DATAPATH}).\r\n\033[0m"
+  printf "\r\n\033[1;31mRun ./testdeployment.sh to rerun the tests.\033[0m"
+  printf "\r\n\033[1;31mRun ./stop.sh to stop cyphernode.\r\n\033[0m"
   exit 1
 fi
 

cyphernodeconf_docker/templates/installer/testfeatures.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-apk add --update --no-cache openssl curl jq coreutils > /dev/null
+apk add --update --no-cache openssl curl jq coreutils postgresql > /dev/null
 
 . /gatekeeper/keys.properties
 
@@ -72,7 +72,7 @@ checkpycoin() {
   echo -en "\r\n\e[1;36mTesting Pycoin... " > /dev/console
   local rc
 
-  rc=$(curl -H "Content-Type: application/json" -d "{\"pub32\":\"upub5GtUcgGed1aGH4HKQ3vMYrsmLXwmHhS1AeX33ZvDgZiyvkGhNTvGd2TA5Lr4v239Fzjj4ZY48t6wTtXUy2yRgapf37QHgt6KWEZ6bgsCLpb\",\"path\":\"0/25-30\"}" -s -o /dev/null -w "%{http_code}" http://proxy:8888/derivepubpath)
+  rc=$(curl -H "Content-Type: application/json" -d "{\"pub32\":\"upub5GtUcgGed1aGH4HKQ3vMYrsmLXwmHhS1AeX33ZvDgZiyvkGhNTvGd2TA5Lr4v239Fzjj4ZY48t6wTtXUy2yRgapf37QHgt6KWEZ6bgsCLpb\",\"path\":\"0/25-30\"}" -s -o /dev/null -w "%{http_code}" http://pycoin:7777/derive)
   [ "${rc}" -ne "200" ] && return 100
 
   echo -e "\e[1;36mPycoin rocks!" > /dev/console
@@ -85,8 +85,9 @@ checkpostgres() {
   local rc
 
   # getbatcher needs the database to return correctly...
-  rc=$(curl -s -o /dev/null -w "%{http_code}" http://proxy:8888/getbatcher)
+#  rc=$(curl -s -o /dev/null -w "%{http_code}" http://proxy:8888/getbatcher)
-  [ "${rc}" -ne "200" ] && return 105
+  pg_isready -h postgres
+  [ "${?}" -ne "0" ] && return 105
 
   echo -e "\e[1;36mPostgres rocks!" > /dev/console
 
@@ -110,7 +111,9 @@ checknotifier() {
   local response
   local returncode
 
-  response=$(mosquitto_rr -h broker -W 15 -t notifier -e "response/$$" -m "{\"response-topic\":\"response/$$\",\"cmd\":\"web\",\"url\":\"http://proxy:8888/helloworld\",\"tor\":false}")
+  nc -vlp1111 -e sh -c 'echo -en "HTTP/1.1 200 OK\\r\\n\\r\\n" ; date >&2 ; timeout 1 tee /dev/tty | cat ; ' &
+  # response=$(mosquitto_rr -h broker -W 15 -t notifier -e "response/$$" -m "{\"response-topic\":\"response/$$\",\"cmd\":\"web\",\"url\":\"http://proxy:8888/helloworld\",\"tor\":false}")
+  response=$(mosquitto_rr -h broker -W 15 -t notifier -e "response/$$" -m "{\"response-topic\":\"response/$$\",\"cmd\":\"web\",\"url\":\"http://$(hostname):1111/notifiertest\",\"tor\":false}")
   returncode=$?
   [ "${returncode}" -ne "0" ] && return 115
   http_code=$(echo "${response}" | jq -r ".http_code")
@@ -125,7 +128,8 @@ checkots() {
   echo -en "\r\n\e[1;36mTesting OTSclient... " > /dev/console
   local rc
 
-  rc=$(curl -s -H "Content-Type: application/json" -d '{"hash":"123","callbackUrl":"http://callback"}' http://proxy:8888/ots_stamp)
+  # rc=$(curl -s -H "Content-Type: application/json" -d '{"hash":"123","callbackUrl":"http://callback"}' http://proxy:8888/ots_stamp)
+  rc=$(curl -s otsclient:6666/stamp/123)
   echo "${rc}" | grep "Invalid hash 123 for sha256" > /dev/null
   [ "$?" -ne "0" ] && return 200
 
@@ -301,7 +305,7 @@ if [ "${returncode}" -ne "0" ]; then
   echo -e "\e[1;31mCyphernode could not fully start properly within delay." > /dev/console
   status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"proxy\") | .active")
   if [ "${status}" = "false" ]; then
-    echo -e "\e[1;31mThe Proxy, the main Cyphernode's component, is not responding.  We will only test the gatekeeper if its container is up, but you'll see errors for the other components.  Please check the logs." > /dev/console
+    echo -e "\r\n\e[1;31mThe Proxy, the main Cyphernode's component, is not responding.  You'll see errors for the other components.  Please check the logs." > /dev/console
     workingproxy="false"
   fi
 else
@@ -323,8 +327,26 @@ fi
 # PROXY                     #
 #############################
 
+if [ ! -f /container_monitor/proxy_dbfailed ]; then
   echo -e "\r\n\e[1;36mWaiting for Proxy to be ready... " > /dev/console
   timeout_feature '[ -f "/container_monitor/proxy_ready" ]' 300
+  returncode=$?
+  if [ "${returncode}" -ne "0" ]; then
+    echo -e "\r\n\e[1;31mThe proxy is still not ready.  It may be migrating large quantity of data?  Please check the logs for more details." > /dev/console
+    workingproxy="false"
+  fi
+fi
+if [ -f /container_monitor/proxy_dbfailed ]; then
+  echo -e "\r\n\e[1;31mThe proxy's database migration failed.  Please check proxy.log for more details." > /dev/console
+  workingproxy="false"
+fi
+
+if [ "${workingproxy}" = "false" ]; then
+  echo -e "\r\n\e[1;31mThe Proxy, the main Cyphernode's component, is not ready.  Cyphernode can't be run without the proxy component." > /dev/console
+  echo -e "\r\n\e[1;31mThe other components will fail next, this is normal." > /dev/console
+fi
+
+result="${containers},\"features\":[{\"coreFeature\":true,\"name\":\"proxy\",\"working\":${workingproxy}}"
 
 #############################
 # POSTGRES                  #
@@ -345,9 +367,9 @@ result="${result}$(feature_status ${returncode} 'Postgres error!')}"
 # GATEKEEPER                #
 #############################
 
-result="${containers},\"features\":[{\"coreFeature\":true, \"name\":\"proxy\",\"working\":${workingproxy}}, {\"coreFeature\":true, \"name\":\"gatekeeper\",\"working\":"
+result="${result},{\"coreFeature\":true,\"name\":\"gatekeeper\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"gatekeeper\") | .active")
-if [ "${status}" = "true" ]; then
+if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
   timeout_feature checkgatekeeper
   returncode=$?
 else
@@ -423,12 +445,11 @@ result="${result}$(feature_status ${returncode} 'OTSclient error!')}"
 # TOR                       #
 #############################
 
-echo -e "\r\n\e[1;36mWaiting for Tor to be ready... " > /dev/console
-timeout_feature '[ -f "/container_monitor/tor_ready" ]'
-
 result="${result},{\"coreFeature\":false,\"name\":\"tor\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"tor\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
+  echo -e "\r\n\e[1;36mWaiting for Tor to be ready... " > /dev/console
+  timeout_feature '[ -f "/container_monitor/tor_ready" ]'
   timeout_feature checktor
   returncode=$?
 else
@@ -442,12 +463,11 @@ result="${result}$(feature_status ${returncode} 'Tor error!')}"
 # BITCOIN                   #
 #############################
 
-echo -e "\r\n\e[1;36mWaiting for Bitcoin Core to be ready... " > /dev/console
-timeout_feature '[ -f "/container_monitor/bitcoin_ready" ]'
-
 result="${result},{\"coreFeature\":true,\"name\":\"bitcoin\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"bitcoin\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
+  echo -e "\r\n\e[1;36mWaiting for Bitcoin Core to be ready... " > /dev/console
+  timeout_feature '[ -f "/container_monitor/bitcoin_ready" ]'
   timeout_feature checkbitcoinnode
   returncode=$?
 else
@@ -461,12 +481,11 @@ result="${result}$(feature_status ${returncode} 'Bitcoin error!')}"
 # LIGHTNING                 #
 #############################
 
-echo -e "\r\n\e[1;36mWaiting for C-Lightning to be ready... " > /dev/console
-timeout_feature '[ -f "/container_monitor/lightning_ready" ]'
-
 result="${result},{\"coreFeature\":false,\"name\":\"lightning\",\"working\":"
 status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"lightning\") | .active")
 if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
+  echo -e "\r\n\e[1;36mWaiting for C-Lightning to be ready... " > /dev/console
+  timeout_feature '[ -f "/container_monitor/lightning_ready" ]'
   timeout_feature checklnnode
   returncode=$?
 else

dist/setup.sh

@@ -16,6 +16,20 @@
 #
 # Let's say if timer > 2, we're on a slow machine.
 
+# At first we tried using uname -m te determine slow devices, but:
+#
+# uname -m result:
+# RPi2: armv7l
+# RPi3: armv7l
+# RPi4 on 32-bit OS: armv7l
+# RPi4 on 64-bit OS: aarch64
+# RockPi: aarch64
+# Apple M1: arm64
+# Intel 64: x86_64#
+#
+# There are a ton of other possible values... and can't rely on them to detect
+# a slow device.
+
 # FROM: https://stackoverflow.com/questions/5195607/checking-bash-exit-status-of-several-commands-efficiently
 # Use step(), try(), and next() to perform a series of commands and print
 # [  OK  ] or [FAILED] at the end. The step as a whole fails if any individual

notifier_docker/script/requesthandler.sh

@@ -35,5 +35,6 @@ main() {
 }
 
 main
+returncode=$?
 trace "[requesthandler] exiting"
-exit $?
+exit ${returncode}

otsclient_docker/script/requesthandler.sh

@@ -97,8 +97,10 @@ main()
     fi
   done
   trace "[main] exiting"
-  return 0
+  return ${returncode}
 }
 
 main
-exit $?
+returncode=$?
+trace "[requesthandler] exiting"
+exit ${returncode}

proxy_docker/app/data/cyphernode.postgresql

@@ -136,8 +136,8 @@ CREATE TABLE cyphernode_props (
 CREATE INDEX idx_cp_property ON cyphernode_props (property);
 CREATE UNIQUE INDEX idx_cp_propval ON cyphernode_props (property, value);
 
-INSERT INTO cyphernode_props (property, value) VALUES ('version', '0.1');
+INSERT INTO cyphernode_props (id, property, value) VALUES (1, 'version', '0.1');
-INSERT INTO cyphernode_props (property, value) VALUES ('pay_index', '0');
+INSERT INTO cyphernode_props (id, property, value) VALUES (2, 'pay_index', '0');
 
 CREATE TABLE ln_invoice (
   id SERIAL PRIMARY KEY,

proxy_docker/app/data/sqlmigrate20210808_0.7.0-0.8.0.sh

@@ -1,14 +1,19 @@
 #!/bin/sh
 
-echo "Checking for labels for watched addresses support in DB..."
+. ./trace.sh
+
+trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Checking for labels for watched addresses support in DB..."
 count=$(sqlite3 $DB_FILE "select count(*) from pragma_table_info('watching') where name='label'")
 if [ "${count}" -eq "0" ]; then
   # label not there, we have to migrate
-	echo "Migrating database for labels for watched addresses support..."
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Migrating database for labels for watched addresses support..."
-  echo "Backing up current DB..."
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Backing up current DB..."
   cp  $DB_FILE $DB_FILE-sqlmigrate20210808_0.7.0-0.8.0
-  echo "Altering DB..."
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Altering DB..."
   cat sqlmigrate20210808_0.7.0-0.8.0.sql | sqlite3 $DB_FILE
+  returncode=$?
+  trace_rc ${returncode}
+  exit ${returncode}
 else
-	echo "Database labels for watched addresses support migration already done, skipping!"
+  trace "[sqlmigrate20210808_0.7.0-0.8.0.sh] Database labels for watched addresses support migration already done, skipping!"
 fi

proxy_docker/app/data/sqlmigrate20210928_0.7.0-0.8.0.sh

@@ -1,14 +1,19 @@
 #!/bin/sh
 
-echo "Checking for new indexes in DB..."
+. ./trace.sh
+
+trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Checking for new indexes in DB..."
 sqlite3 $DB_FILE ".indexes" | grep "idx_watching_watching" > /dev/null
 if [ "$?" -eq "1" ]; then
   # idx_watching_watching index not found
-  echo "Migrating database with new indexes..."
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Migrating database with new indexes..."
-  echo "Backing up current DB..."
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Backing up current DB..."
   cp $DB_FILE $DB_FILE-sqlmigrate20210928_0.7.0-0.8.0
-  echo "Altering DB..."
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] Altering DB..."
   cat sqlmigrate20210928_0.7.0-0.8.0.sql | sqlite3 $DB_FILE
+  returncode=$?
+  trace_rc ${returncode}
+  exit ${returncode}
 else
-  echo "New indexes migration already done, skipping!"
+  trace "[sqlmigrate20210928_0.7.0-0.8.0.sh] New indexes migration already done, skipping!"
 fi

proxy_docker/app/data/sqlmigrate20211105_0.7.0-0.8.0.sh

@@ -1,26 +1,39 @@
 #!/bin/sh
 
-date
+. ./trace.sh
-echo "Waiting for postgres to be ready..."
-(while true ; do psql -h postgres -U cyphernode -c "select 1;" ; [ "$?" -eq "0" ] && break ; sleep 10; done) &
-wait
 
-echo "Checking if postgres is setup..."
+trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Waiting for postgres to be ready..."
+while true ; do psql -h postgres -U cyphernode -c "select 1;" ; [ "$?" -eq "0" ] && break ; sleep 10; done
+
+trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Checking if postgres is setup..."
 psql -h postgres -U cyphernode -c "\d" | grep "cyphernode_props" > /dev/null
 if [ "$?" -eq "1" ]; then
   # if cyphernode_props table doesn't exist, it's probably because database hasn't been setup yet
-  echo "Creating postgres database..."
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Creating postgres database..."
   psql -h postgres -f cyphernode.postgresql -U cyphernode
+  returncode=$?
+  trace_rc ${returncode}
+  [ "${returncode}" -eq "0" ] || exit ${returncode}
+else
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] PostgreSQL database already created, skipping!"
+fi
 
-  date
+trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Checking if postgres is loaded/imported..."
-  echo "Extracting and converting sqlite3 data..."
+lastval=$(psql -qAtX -h postgres -U cyphernode -c "select last_value from pg_sequences where sequencename='cyphernode_props_id_seq'")
-  date
+returncode=$?
+if [ -z "${lastval}" ] || [ "${lastval}" -lt "2" ]; then
+  # if cyphernode_props_id_seq isn't set, it's probably because database hasn't been loaded/imported yet
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Extracting and converting sqlite3 data..."
   cat sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extract.sql | sqlite3 $DB_FILE
+  returncode=$?
+  trace_rc ${returncode}
+  [ "${returncode}" -eq "0" ] || exit ${returncode}
+
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Creating import file for postgres..."
   mv sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql ${DB_PATH}/
   sed -ie 's/^\(INSERT.*\);$/\1 ON CONFLICT DO NOTHING;/g' ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql
 
-  date
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Appending postgresql sequence creation..."
-  echo "...appending postgresql sequences..."
   echo "
 select setval('cyphernode_props_id_seq',  (SELECT MAX(id) FROM cyphernode_props));
 select setval('ln_invoice_id_seq',  (SELECT MAX(id) FROM ln_invoice));
@@ -34,10 +47,11 @@ select setval('batcher_id_seq',  (SELECT MAX(id) FROM batcher));
 commit;
 " >> ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql
 
-  date
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] Importing sqlite3 data into postgresql..."
-  echo "Importing sqlite3 data into postgresql..."
+  psql -v ON_ERROR_STOP=on -h postgres -f ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql -U cyphernode
-  psql -h postgres -f ${DB_PATH}/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extracted-data.sql -U cyphernode
+  returncode=$?
-  date
+  trace_rc ${returncode}
+  [ "${returncode}" -eq "0" ] || exit ${returncode}
 else
-  echo "New indexes migration already done, skipping!"
+  trace "[sqlmigrate20211105_0.7.0-0.8.0.sh] PostgreSQL database already loaded, skipping!"
 fi

proxy_docker/app/data/sqlmigrate20211105_0.7.0-0.8.0_sqlite3-extract.sql

@@ -17,11 +17,10 @@ select id,address,amount,tx_id,inserted_ts,webhook_url,case when calledback=1 th
 select id,txid,case when watching=1 then 'TRUE' else 'FALSE' end as watching,callback1conf,case when calledback1conf=1 then 'TRUE' else 'FALSE' end as calledback1conf,callbackxconf,case when calledbackxconf=1 then 'TRUE' else 'FALSE' end as calledbackxconf,nbxconf,inserted_ts from watching_by_txid;
 .mode insert stamp
 select id,hash,callbackUrl,case when requested=1 then 'TRUE' else 'FALSE' end as requested,case when upgraded=1 then 'TRUE' else 'FALSE' end as upgraded,case when calledback=1 then 'TRUE' else 'FALSE' end as calledback,inserted_ts from stamp;
+.mode insert ln_invoice
+select id,label,bolt11,payment_hash,msatoshi,status,pay_index,msatoshi_received,paid_at,description,expires_at,callback_url,case when calledback=1 then 'TRUE' else 'FALSE' end as calledback,case when callback_failed=1 then 'TRUE' else 'FALSE' end as callback_failed,inserted_ts from ln_invoice;
 -- cyphernode_props rows were already inserted in db creation, let's update them here
 .headers off
 .mode list cyphernode_props
 select 'update cyphernode_props set value=''' || value || ''', inserted_ts=''' || inserted_ts || ''' where id=' || id || ';' from cyphernode_props;
-.headers on
-.mode insert ln_invoice
-select id,label,bolt11,payment_hash,msatoshi,status,pay_index,msatoshi_received,paid_at,description,expires_at,callback_url,case when calledback=1 then 'TRUE' else 'FALSE' end as calledback,case when callback_failed=1 then 'TRUE' else 'FALSE' end as callback_failed,inserted_ts from ln_invoice;
 .quit

proxy_docker/app/script/computefees.sh

@@ -11,7 +11,7 @@ compute_fees() {
     trace "[compute_fees]  pruned=${pruned}"
     # We want null instead of 0.00000000 in this case.
     echo "null"
-    exit 0
+    return
   fi
 
   local txid=${1}

proxy_docker/app/script/requesthandler.sh

@@ -771,9 +771,10 @@ main() {
     fi
   done
   trace "[main] exiting"
-  return 0
+  return ${returncode}
 }
 
 main
+returncode=$?
 trace "[requesthandler] exiting"
-exit $?
+exit ${returncode}

proxy_docker/app/script/startproxy.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+. ./trace.sh
+
 trim() {
   echo -e "$1" | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//'
 }
@@ -7,12 +9,12 @@ trim() {
 createCurlConfig() {
 
   if [[ ''$1 == '' ]]; then
-    echo "Missing file name: Check your *_BTC_NODE_RPC_CFG"
+    trace "[startproxy] Missing file name: Check your *_BTC_NODE_RPC_CFG"
     return
   fi
 
   if [[ ''$2 == '' ]]; then
-    echo "Missing content: Check your *_BTC_NODE_RPC_USER"
+    trace "[startproxy] Missing content: Check your *_BTC_NODE_RPC_USER"
     return
   fi
 
@@ -21,17 +23,45 @@ createCurlConfig() {
 
 }
 
+if [ -e ${DB_PATH}/.dbfailed ]; then
+  touch /container_monitor/proxy_dbfailed
+  trace "[startproxy] A previous database creation/migration failed.  Stopping."
+  trace "[startproxy] A file called .dbfailed has been created.  Fix the migration errors, remove .dbfailed and retry."
+  trace "[startproxy] Exiting."
+  sleep 30
+  exit 1
+else
+  rm -f /container_monitor/proxy_dbfailed
+fi
+
 if [ ! -e ${DB_FILE} ]; then
-  echo "DB not found, creating..."
+  trace "[startproxy] DB not found, creating..."
   cat cyphernode.sql | sqlite3 $DB_FILE
   psql -h postgres -f cyphernode.postgresql -U cyphernode
+  returncode=$?
+  trace_rc ${returncode}
 else
-  echo "DB found, migrating..."
+  trace "[startproxy] DB found, migrating..."
   for script in sqlmigrate*.sh; do
     sh $script
+    returncode=$?
+    trace_rc ${returncode}
+    if [ "${returncode}" -ne "0" ]; then
+      break
+    fi
   done
 fi
 
+if [ "${returncode}" -ne "0" ]; then
+  touch ${DB_PATH}/.dbfailed
+  touch /container_monitor/proxy_dbfailed
+  trace "[startproxy] Database creation/migration failed.  Stopping."
+  trace "[startproxy] A file called .dbfailed has been created in your proxy datapath.  Fix the migration errors, remove .dbfailed and retry."
+  trace "[startproxy] Exiting."
+  sleep 30
+  exit ${returncode}
+fi
+
 chmod 0600 $DB_FILE
 
 createCurlConfig ${WATCHER_BTC_NODE_RPC_CFG} ${WATCHER_BTC_NODE_RPC_USER}

proxy_docker/app/script/trace.sh

@@ -1,7 +1,6 @@
 #!/bin/sh
 
-trace()
+trace() {
-{
   if [ -n "${TRACING}" ]; then
     local str="$(date -Is) $$ ${1}"
     echo "${str}" 1>&2
@@ -9,8 +8,7 @@ trace()
   fi
 }
 
-trace_rc()
+trace_rc() {
-{
   if [ -n "${TRACING}" ]; then
     local str="$(date -Is) $$ Last return code: ${1}"
     echo "${str}" 1>&2