Cleanups, fix pid1 in containers, use proxy_ready monitor

api_auth_docker/entrypoint.sh

@@ -1,17 +1,16 @@
 #!/bin/bash
 
-user='nginx'
+while [ ! -f "/container_monitor/proxy_ready" ]; do echo "proxy not ready" ; sleep 10 ; done
+
+echo "proxy ready"
 
 if [[ $1 ]]; then
-	IFS=':' read -ra arr <<< "$1"
+	user=$(echo $1 | cut -d ':' -f 1)
-
+else
-	if [[ ${arr[0]} ]]; then
+	user='nginx'
-		user=${arr[0]};
-	fi
-
 fi
 
 spawn-fcgi -M 0660 -s /var/run/fcgiwrap.socket -u $user -g nginx -U $user -- `which fcgiwrap`
 chmod -R g+rw /var/run/fcgiwrap.socket /etc/nginx/conf.d/*
 chown -R :nginx /etc/nginx/conf.d/*
-nginx -g "daemon off;"
+exec nginx -g "daemon off;"

cyphernodeconf_docker/templates/installer/docker/docker-compose.yaml

@@ -46,8 +46,8 @@ services:
       - "<%= tor_datapath %>:/tor"
       - container_monitor:/container_monitor
     healthcheck:
-      test: chown $USER /container_monitor && su-exec $USER sh -c 'tor-resolve torproject.org && touch /container_monitor/tor_ready && chown $USER /container_monitor/tor_ready || rm -f /container_monitor/tor_ready'
+      test: chown -R $USER /container_monitor && su-exec $USER sh -c 'tor-resolve torproject.org && touch /container_monitor/tor_ready || rm -f /container_monitor/tor_ready'
-      interval: 20s
+      interval: 30s
       timeout: 10s
       retries: 10
     networks:
@@ -86,8 +86,8 @@ services:
       - "<%= bitcoin_datapath %>/bitcoin-client.conf:/.bitcoin/bitcoin.conf:ro"
       - container_monitor:/container_monitor
     healthcheck:
-      test: chown $USER /container_monitor && su-exec $USER sh -c 'lightning-cli getinfo && touch /container_monitor/lightning_ready && chown $USER /container_monitor/lightning_ready || rm -f /container_monitor/lightning_ready'
+      test: chown -R $USER /container_monitor && su-exec $USER sh -c 'lightning-cli getinfo && touch /container_monitor/lightning_ready || rm -f /container_monitor/lightning_ready'
-      interval: 20s
+      interval: 30s
       timeout: 10s
       retries: 10
     stop_grace_period: 30s
@@ -130,8 +130,8 @@ services:
       - "<%= bitcoin_datapath %>/createWallets.sh:/.bitcoin/createWallets.sh:ro"
       - container_monitor:/container_monitor
     healthcheck:
-      test: chown $USER /container_monitor && su-exec $USER sh -c 'bitcoin-cli echo && touch /container_monitor/bitcoin_ready || rm -f /container_monitor/bitcoin_ready'
+      test: chown -R $USER /container_monitor && su-exec $USER sh -c 'bitcoin-cli echo && touch /container_monitor/bitcoin_ready || rm -f /container_monitor/bitcoin_ready'
-      interval: 20s
+      interval: 30s
       timeout: 10s
       retries: 10
     stop_grace_period: 30s
@@ -204,6 +204,13 @@ services:
       <% if ( features.indexOf('tor') !== -1 ) { %>
       - "<%= tor_datapath %>:/proxy/tor"
       <% } %>
+      - container_monitor:/container_monitor
+    healthcheck:
+      test: chown -R $USER /container_monitor && su-exec $USER sh -c 'curl localhost:8888/helloworld && touch /container_monitor/proxy_ready || rm -f /container_monitor/proxy_ready'
+      interval: 30s
+      timeout: 10s
+      retries: 10
+    stop_grace_period: 30s
     networks:
       - cyphernodenet
     depends_on:
@@ -284,6 +291,7 @@ services:
     command: $USER ./startnotifier.sh
     <% if ( features.indexOf('tor') !== -1 ) { %>
     environment:
+      - "TRACING=1"
       - "TOR_HOST=tor"
       - "TOR_PORT=9050"
     <% } %>
@@ -407,6 +415,7 @@ services:
       - "<%= gatekeeper_datapath %>/installation.json:/etc/nginx/conf.d/s/stats/installation.json"
       - "<%= gatekeeper_datapath %>/client.7z:/etc/nginx/conf.d/s/stats/client.7z"
       - "<%= gatekeeper_datapath %>/config.7z:/etc/nginx/conf.d/s/stats/config.7z"
+      - container_monitor:/container_monitor
     networks:
       - cyphernodenet
       - cyphernodeappsnet

cyphernodeconf_docker/templates/installer/start.sh

@@ -68,11 +68,13 @@ docker-compose -f $current_path/docker-compose.yaml up -d --remove-orphans
 
 start_apps
 
-export ARCH=$(uname -m)
+printf "\r\nDetermining the speed of your machine..."
-case "${ARCH}" in arm*)
+speedseconds=$(bash -c ' : {1..500000} ; echo $SECONDS')
-  printf "\r\n\033[1;31mSince we're on a slow RPi, let's give Docker 60 more seconds before performing our tests...\033[0m\r\n\r\n"
+if [[ $speedseconds > 2 ]]; then
+  printf "\r\n\033[1;31mSince we're on a slow computer, let's give Docker 60 more seconds before performing our tests...\033[0m\r\n\r\n"
   sleep 60
-;;
+else
-esac
+  printf " It's pretty fast!\r\n"
+fi
 
 . ./testdeployment.sh

cyphernodeconf_docker/templates/installer/testfeatures.sh

@@ -232,7 +232,7 @@ checkservice() {
 
 timeout_feature() {
   local interval=15
-  local totaltime=120
+  local totaltime=${2:-120}
   local testwhat=${1}
   local returncode
   local endtime=$(($(date +%s) + ${totaltime}))
@@ -319,6 +319,28 @@ fi
 #    { "name": "lightning", "working":true },
 #  ]
 
+#############################
+# PROXY                     #
+#############################
+
+echo -e "\r\n\e[1;36mWaiting for Proxy to be ready... " > /dev/console
+timeout_feature '[ -f "/container_monitor/proxy_ready" ]' 300
+
+#############################
+# POSTGRES                  #
+#############################
+
+result="${result},{\"coreFeature\":true, \"name\":\"postgres\",\"working\":"
+status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"postgres\") | .active")
+if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
+  timeout_feature checkpostgres
+  returncode=$?
+else
+  returncode=1
+fi
+finalreturncode=$((${returncode} | ${finalreturncode}))
+result="${result}$(feature_status ${returncode} 'Postgres error!')}"
+
 #############################
 # GATEKEEPER                #
 #############################
@@ -379,21 +401,6 @@ fi
 finalreturncode=$((${returncode} | ${finalreturncode}))
 result="${result}$(feature_status ${returncode} 'Pycoin error!')}"
 
-#############################
-# POSTGRES                  #
-#############################
-
-result="${result},{\"coreFeature\":true, \"name\":\"postgres\",\"working\":"
-status=$(echo "{${containers}}" | jq ".containers[] | select(.name == \"postgres\") | .active")
-if [[ "${workingproxy}" = "true" && "${status}" = "true" ]]; then
-  timeout_feature checkpostgres
-  returncode=$?
-else
-  returncode=1
-fi
-finalreturncode=$((${returncode} | ${finalreturncode}))
-result="${result}$(feature_status ${returncode} 'Postgres error!')}"
-
 <% if (features.indexOf('otsclient') != -1) { %>
 #############################
 # OTSCLIENT                 #
@@ -470,6 +477,8 @@ result="${result}$(feature_status ${returncode} 'Lightning error!')}"
 
 <% } %>
 
+#############################
+
 result="{${result}]}"
 
 echo "${result}" > /gatekeeper/installation.json

dist/setup.sh

@@ -1,17 +1,20 @@
 #!/bin/bash
 
-### Execute this on a freshly install ubuntu luna node
+# This is where everything is configured.
-# curl -fsSL get.docker.com -o get-docker.sh
-# sh get-docker.sh
-# sudo usermod -aG docker $USER
-## logout and relogin
-# git clone --branch features/install --recursive https://github.com/schulterklopfer/cyphernode.git
-# sudo curl -L "https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-# sudo chmod +x /usr/local/bin/docker-compose
-# cd cyphernode
-# ./setup.sh -ci
-# docker-compose -f docker-compose.yaml up [-d]
 
+# To determine speed of machine...
+#
+# bash -c ' : {1..500000} ; echo $SECONDS'
+#
+# MBP M1: 0
+# MBP Intel: 0
+# x86_64 avg machine: 0
+# RockPi Debian 64-bits: 1
+# RPi4 RaspiOS 64-bits: 1
+# RPi3 RaspiOS 32-bits: 4
+# RPi2 RaspiOS 32-bits: 7
+#
+# Let's say if timer > 2, we're on a slow machine.
 
 # FROM: https://stackoverflow.com/questions/5195607/checking-bash-exit-status-of-several-commands-efficiently
 # Use step(), try(), and next() to perform a series of commands and print
@@ -142,9 +145,6 @@ configure() {
     recreate=" recreate"
   fi
 
-
-
-  local arch=$(uname -m)
   local pw_env=''
   local interactive=''
   local gen_options=''
@@ -157,11 +157,12 @@ configure() {
     pw_env=" -e CFG_PASSWORD=$CFG_PASSWORD"
   fi
 
-
+  echo "\nDetermining the speed of your machine..."
-  if [[ $arch =~ ^arm ]]; then
+  local speedseconds=$(bash -c ' : {1..500000} ; echo $SECONDS')
-    clear && echo "Thinking. This may take a while, since I'm a Raspberry PI and my brain is so tiny. :("
+  if [[ $speedseconds > 2 ]]; then
+    clear && echo "This may take a while, since it seems we're running on a slow machine."
   else
-    clear && echo "Thinking..."
+    clear && echo "Fast machine..."
   fi
 
   # before starting a new cyphernodeconf, kill all the others
@@ -349,14 +350,6 @@ compare_bitcoinconf() {
 }
 
 install_docker() {
-  local archpath=$(uname -m)
-
-  # compat mode for SatoshiPortal repo
-  # TODO: add more mappings?
-  if [[ $archpath == 'armv7l' ]]; then
-    archpath="rpi"
-  fi
-
   if [ ! -d $GATEKEEPER_DATAPATH ]; then
     step "   create $GATEKEEPER_DATAPATH"
     sudo_if_required mkdir -p $GATEKEEPER_DATAPATH

notifier_docker/script/requesthandler.sh

@@ -34,8 +34,6 @@ main() {
   done
 }
 
-export TRACING=1
-
 main
 trace "[requesthandler] exiting"
 exit $?

notifier_docker/script/startnotifier.sh

@@ -2,4 +2,6 @@
 
 . ./trace.sh
 
-mosquitto_sub -h broker -t notifier | ./requesthandler.sh
+trace "Starting mosquitto and subscribing to the notifier topic..."
+
+exec sh -c 'mosquitto_sub -h broker -t notifier | ./requesthandler.sh'

otsclient_docker/script/requesthandler.sh

@@ -64,33 +64,35 @@ main()
           # GET http://192.168.111.152:8080/stamp/1ddfb769eb0b8876bc570e25580e6a53afcf973362ee1ee4b54a807da2e5eed7
 
           response=$(stamp $(echo "${line}" | cut -d ' ' -f2 | cut -d '/' -f3))
-          response_to_client "${response}" ${?}
+          returncode=$?
-          break
           ;;
         upgrade)
           # GET http://192.168.111.152:8080/upgrade/1ddfb769eb0b8876bc570e25580e6a53afcf973362ee1ee4b54a807da2e5eed7
 
           response=$(upgrade $(echo "${line}" | cut -d ' ' -f2 | cut -d '/' -f3))
-          response_to_client "${response}" ${?}
+          returncode=$?
-          break
           ;;
         verify)
           # POST http://192.168.111.152:8080/verify
           # BODY {"hash":"1ddfb769eb0b8876bc570e25580e6a53afcf973362ee1ee4b54a807da2e5eed7","base64otsfile":"AE9wZW5UaW1lc3RhbXBzAABQcm9vZ...gABYiWDXPXGQEDxNch"}
 
           response=$(verify "${line}")
-          response_to_client "${response}" ${?}
+          returncode=$?
-          break
           ;;
         info)
           # POST http://192.168.111.152:8080/info
           # BODY {"base64otsfile":"AE9wZW5UaW1lc3RhbXBzAABQcm9vZ...gABYiWDXPXGQEDxNch"}
 
           response=$(info "${line}")
-          response_to_client "${response}" ${?}
+          returncode=$?
-          break
+          ;;
+        *)
+          response='{"error": {"code": -32601, "message": "Method not found"}, "id": "1"}'
+          returncode=1
           ;;
       esac
+      response=$(echo "${response}" | jq -Mc)
+      response_to_client "${response}" ${returncode}
       break
     fi
   done
@@ -98,7 +100,5 @@ main()
   return 0
 }
 
-export TRACING
-
 main
 exit $?

otsclient_docker/script/startotsclient.sh

@@ -1,6 +1,3 @@
 #!/bin/sh
 
-export TRACING
+exec nc -vlkp${OTSCLIENT_LISTENING_PORT} -e ./requesthandler.sh
-export OTSCLIENT_LISTENING_PORT
-
-nc -vlkp${OTSCLIENT_LISTENING_PORT} -e ./requesthandler.sh

proxy_docker/app/script/requesthandler.sh

@@ -76,7 +76,7 @@ main() {
       case "${cmd}" in
         helloworld)
           # GET http://192.168.111.152:8080/helloworld
-          response="Hello, world!"
+          response='{"hello":"world"}'
           returncode=0
           # response_to_client "Hello, world!" 0
           # break
@@ -774,11 +774,6 @@ main() {
   return 0
 }
 
-export NODE_RPC_URL=$BTC_NODE_RPC_URL
-export TRACING
-export DB_PATH
-export DB_FILE
-
 main
 trace "[requesthandler] exiting"
 exit $?

proxy_docker/app/script/sendtobitcoinnode.sh

@@ -5,7 +5,7 @@
 send_to_watcher_node() {
   trace "Entering send_to_watcher_node()..."
   local node_payload
-  node_payload="$(send_to_bitcoin_node ${WATCHER_NODE_RPC_URL}/${WATCHER_BTC_NODE_DEFAULT_WALLET} ${WATCHER_NODE_RPC_CFG} $@)"
+  node_payload="$(send_to_bitcoin_node ${WATCHER_BTC_NODE_RPC_URL}/${WATCHER_BTC_NODE_DEFAULT_WALLET} ${WATCHER_BTC_NODE_RPC_CFG} $@)"
   local returncode=$?
   trace_rc ${returncode}
   if [ "${returncode}" -ne 0 ]; then
@@ -22,7 +22,7 @@ send_to_watcher_node() {
 send_to_xpub_watcher_wallet() {
   trace "Entering send_to_xpub_watcher_wallet()..."
 
-  send_to_bitcoin_node ${WATCHER_NODE_RPC_URL}/${WATCHER_BTC_NODE_XPUB_WALLET} ${WATCHER_NODE_RPC_CFG} $@
+  send_to_bitcoin_node ${WATCHER_BTC_NODE_RPC_URL}/${WATCHER_BTC_NODE_XPUB_WALLET} ${WATCHER_BTC_NODE_RPC_CFG} $@
   local returncode=$?
   trace_rc ${returncode}
   return ${returncode}
@@ -33,7 +33,7 @@ send_to_watcher_node_wallet() {
   local walletname=$1
   shift
   trace "[send_to_watcher_node_wallet] walletname=${walletname}"
-  send_to_bitcoin_node ${WATCHER_NODE_RPC_URL}/${walletname} ${WATCHER_NODE_RPC_CFG} $@
+  send_to_bitcoin_node ${WATCHER_BTC_NODE_RPC_URL}/${walletname} ${WATCHER_BTC_NODE_RPC_CFG} $@
   local returncode=$?
   trace_rc ${returncode}
   return ${returncode}
@@ -42,7 +42,7 @@ send_to_watcher_node_wallet() {
 send_to_spender_node()
 {
   trace "Entering send_to_spender_node()..."
-  send_to_bitcoin_node ${SPENDER_NODE_RPC_URL}/${SPENDER_BTC_NODE_DEFAULT_WALLET} ${SPENDER_NODE_RPC_CFG} $@
+  send_to_bitcoin_node ${SPENDER_BTC_NODE_RPC_URL}/${SPENDER_BTC_NODE_DEFAULT_WALLET} ${SPENDER_BTC_NODE_RPC_CFG} $@
   local returncode=$?
   trace_rc ${returncode}
   return ${returncode}

proxy_docker/app/script/startproxy.sh

@@ -1,14 +1,5 @@
 #!/bin/sh
 
-export PROXY_LISTENING_PORT
-export WATCHER_NODE_RPC_URL=$WATCHER_BTC_NODE_RPC_URL
-export SPENDER_NODE_RPC_URL=$SPENDER_BTC_NODE_RPC_URL
-export WATCHER_NODE_RPC_CFG=$WATCHER_BTC_NODE_RPC_CFG
-export SPENDER_NODE_RPC_CFG=$SPENDER_BTC_NODE_RPC_CFG
-export TRACING
-export DB_PATH
-export DB_FILE
-
 trim() {
   echo -e "$1" | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//'
 }
@@ -51,4 +42,4 @@ if [ "${FEATURE_LIGHTNING}" = "true" ]; then
   ./waitanyinvoice.sh &
 fi
 
-nc -vlkp${PROXY_LISTENING_PORT} -e ./requesthandler.sh
+exec nc -vlkp${PROXY_LISTENING_PORT} -e ./requesthandler.sh

proxy_docker/app/tests/test-manage-missed.sh

@@ -38,6 +38,16 @@ exec_in_test_container() {
   docker exec -it tests-manage-missed $@
 }
 
+wait_for_proxy() {
+  trace 1 "\n\n[wait_for_proxy] ${BCyan}Waiting for the proxy to be ready...${Color_Off}\n"
+
+  # First ping the containers to make sure they're up...
+  docker exec -t tests-manage-missed sh -c 'while true ; do ping -c 1 proxy ; [ "$?" -eq "0" ] && break ; sleep 5; done'
+
+  # Now check if the lightning nodes are ready to accept requests...
+  docker exec -t tests-manage-missed sh -c 'while true ; do curl proxy:8888/helloworld ; [ "$?" -eq "0" ] && break ; sleep 5; done'
+}
+
 test_manage_missed_0_conf() {
   # Missed 0-conf:
   # 1. Get new address
@@ -69,8 +79,7 @@ test_manage_missed_0_conf() {
   docker exec -it $(docker ps -q -f "name=cyphernode_bitcoin") bitcoin-cli -rpcwallet=spending01.dat sendtoaddress ${address} 0.0001
   # txid1=$(exec_in_test_container curl -d '{"address":"'${address}'","amount":0.0001}' proxy:8888/spend | jq -r ".txid")
 
-  trace 3 "[test_manage_missed_0_conf] Sleeping for 10 seconds to let the proxy restart..."
+  wait_for_proxy
-  sleep 10
 
   trace 3 "[test_manage_missed_0_conf] Calling executecallbacks..."
   exec_in_test_container curl -s -H "Content-Type: application/json" proxy:8888/executecallbacks
@@ -116,8 +125,7 @@ test_manage_missed_1_conf() {
   trace 3 "[test_manage_missed_1_conf] Mine a new block..."
   mine
 
-  trace 3 "[test_manage_missed_1_conf] Sleeping for 10 seconds to let the proxy restart..."
+  wait_for_proxy
-  sleep 10
 
   trace 3 "[test_manage_missed_1_conf] Calling executecallbacks..."
   exec_in_test_container curl -s -H "Content-Type: application/json" proxy:8888/executecallbacks
@@ -148,8 +156,7 @@ trace 2 "url3=${url3}"
 trace 2 "url4=${url4}"
 
 exec_in_test_container apk add --update curl
-# exec_in_test_container ping -c 5 tests-manage-missed
+
-# exec_in_test_container curl -vd 'toto' ${url1}/allo
 test_manage_missed_0_conf
 test_manage_missed_1_conf
 

pycoin_docker/script/requesthandler.sh

@@ -8,8 +8,7 @@
 . ./responsetoclient.sh
 . ./trace.sh
 
-main()
+main() {
-{
   trace "Entering main()..."
 
   local step=0
@@ -67,10 +66,15 @@ main()
           # BODY {"pub32":"vpub5SLqN2bLY4WeZF3kL4VqiWF1itbf3A6oRrq9aPf16AZMVWYCuN9TxpAZwCzVgW94TNzZPNc9XAHD4As6pdnExBtCDGYRmNJrcJ4eV9hNqcv","path":"0/25-30"}
 
           response=$(derive "${line}")
-					response_to_client "${response}" ${?}
+          returncode=$?
-					break
+          ;;
+        *)
+          response='{"error": {"code": -32601, "message": "Method not found"}, "id": "1"}'
+          returncode=1
           ;;
       esac
+      response=$(echo "${response}" | jq -Mc)
+      response_to_client "${response}" ${returncode}
       break
     fi
   done
@@ -78,7 +82,5 @@ main()
   return 0
 }
 
-export TRACING
-
 main
 exit $?

pycoin_docker/script/startpycoin.sh

@@ -1,6 +1,3 @@
 #!/bin/sh
 
-export TRACING
+exec nc -vlkp${PYCOIN_LISTENING_PORT} -e ./requesthandler.sh
-export PYCOIN_LISTENING_PORT
-
-nc -vlkp${PYCOIN_LISTENING_PORT} -e ./requesthandler.sh