From a8f7cfb2ef89ee1df83795b8c36ff7b4c0738c9b Mon Sep 17 00:00:00 2001 From: kexkey Date: Thu, 10 Jan 2019 10:05:24 -0500 Subject: [PATCH 1/5] Added PID in traces --- api_auth_docker/trace.sh | 4 ++-- otsclient_docker/script/trace.sh | 4 ++-- proxy_docker/app/script/trace.sh | 4 ++-- pycoin_docker/script/trace.sh | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api_auth_docker/trace.sh b/api_auth_docker/trace.sh index c2c46ed..a5141fb 100644 --- a/api_auth_docker/trace.sh +++ b/api_auth_docker/trace.sh @@ -3,13 +3,13 @@ trace() { if [ -n "${TRACING}" ]; then - echo "[$(date +%Y-%m-%dT%H:%M:%S%z)] ${1}" 1>&2 + echo "[$(date +%Y-%m-%dT%H:%M:%S%z)] $$ ${1}" 1>&2 fi } trace_rc() { if [ -n "${TRACING}" ]; then - echo "[$(date +%Y-%m-%dT%H:%M:%S%z)] Last return code: ${1}" 1>&2 + echo "[$(date +%Y-%m-%dT%H:%M:%S%z)] $$ Last return code: ${1}" 1>&2 fi } diff --git a/otsclient_docker/script/trace.sh b/otsclient_docker/script/trace.sh index 680f3f2..4c0a1c2 100644 --- a/otsclient_docker/script/trace.sh +++ b/otsclient_docker/script/trace.sh @@ -3,13 +3,13 @@ trace() { if [ -n "${TRACING}" ]; then - echo "$(date -Is) ${1}" 1>&2 + echo "$(date -Is) $$ ${1}" 1>&2 fi } trace_rc() { if [ -n "${TRACING}" ]; then - echo "$(date -Is) Last return code: ${1}" 1>&2 + echo "$(date -Is) $$ Last return code: ${1}" 1>&2 fi } diff --git a/proxy_docker/app/script/trace.sh b/proxy_docker/app/script/trace.sh index 680f3f2..4c0a1c2 100644 --- a/proxy_docker/app/script/trace.sh +++ b/proxy_docker/app/script/trace.sh @@ -3,13 +3,13 @@ trace() { if [ -n "${TRACING}" ]; then - echo "$(date -Is) ${1}" 1>&2 + echo "$(date -Is) $$ ${1}" 1>&2 fi } trace_rc() { if [ -n "${TRACING}" ]; then - echo "$(date -Is) Last return code: ${1}" 1>&2 + echo "$(date -Is) $$ Last return code: ${1}" 1>&2 fi } diff --git a/pycoin_docker/script/trace.sh b/pycoin_docker/script/trace.sh index 680f3f2..4c0a1c2 100644 --- a/pycoin_docker/script/trace.sh +++ b/pycoin_docker/script/trace.sh @@ -3,13 +3,13 @@ trace() { if [ -n "${TRACING}" ]; then - echo "$(date -Is) ${1}" 1>&2 + echo "$(date -Is) $$ ${1}" 1>&2 fi } trace_rc() { if [ -n "${TRACING}" ]; then - echo "$(date -Is) Last return code: ${1}" 1>&2 + echo "$(date -Is) $$ Last return code: ${1}" 1>&2 fi } From 06a9665339fa1e0b9156d4197f53e4b979f90c35 Mon Sep 17 00:00:00 2001 From: Francis Pouliot Date: Sun, 13 Jan 2019 19:42:50 -0500 Subject: [PATCH 2/5] Update README.md --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 6e901f9..a391033 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,38 @@ The docker containers used in this project are hosted at www.bitcoindockers.com The project is in **heavy development** - we are currently looking for review, new features, user feedback and contributors to our roadmap. +# Cyphernode Architecture +Cyphernode is an assembly of Docker containers being called by a request dispatcher. + +The request dispatcher (requesthandler.sh) is the HTTP entry point. +The request dispatcher is stateful: it keeps some data to be more effective on next calls. +The request dispatcher is where Cyphernode scales with new features: add your switch, dispatch requests to your stuff. +We are trying to construct each container so that it can be used separately, as a standalone reusable component. + +Important to us: + +Be as optimized as possible, using Alpine when possible and having the smallest Docker image size possible +Reuse existing software: built-in shell commands, well-established pieces of software, etc. +Use open-source software +Don't reinvent the wheel +Expose the less possible surface +Center element: proxy_docker +The proxy_docker is the container receiving and dispatching calls from clients. When adding a feature to Cyphernode, it is the first part to be modified to integrate the new feature. + +proxy_docker/app/script/requesthandler.sh +You will find in there the switch statement used to dispatch the requests. Just add a case block with your command, using other cases as examples for POST or GET requests. + +proxy_docker/app/config +You will find there config files. config.properties should be used to centralize configs. spender and watcher properties are used to obfuscate credentials on curl calls. + +proxy_docker/app/data +watching.sql contains the data model. Called "watching" because in the beginning of the project, it was only used for watching addresses. Now could be used to index the blockchain (build an explorer) and add more features. + +cron_docker +If you have jobs to be scheduled, use this container. Just add an executable and add it to the crontab. + +Currently used to make sure callbacks have been called for missed transactions. + # About this project - Created and maintained by www.satoshiportal.com From 2db77a9aba2bc438157ede2aacc5dd688f6c052c Mon Sep 17 00:00:00 2001 From: SKP Date: Thu, 17 Jan 2019 18:04:16 +0100 Subject: [PATCH 3/5] Fixed installation directory problems --- build.sh | 38 +++++---- dist/setup.sh | 9 +++ .../generators/app/index.js | 2 + .../generators/app/prompters/999_installer.js | 80 ++++++++++++------- 4 files changed, 82 insertions(+), 47 deletions(-) diff --git a/build.sh b/build.sh index a03dd62..a5deed5 100755 --- a/build.sh +++ b/build.sh @@ -2,6 +2,17 @@ TRACING=1 +# CYPHERNODE VERSION "v0.1" +CONF_VERSION="v0.1" +GATEKEEPER_VERSION="v0.1" +PROXY_VERSION="v0.1" +PROXYCRON_VERSION="v0.1" +OTSCLIENT_VERSION="v0.1" +PYCOIN_VERSION="v0.1" +BITCOIN_VERSION="v0.17.0" +LIGHTNING_VERSION="v0.6.2" +GRAFANA_VERSION="v0.1" + trace() { if [ -n "${TRACING}" ]; then @@ -38,6 +49,7 @@ build_docker_images() { local bitcoin_dockerfile=Dockerfile.amd64 local clightning_dockerfile=Dockerfile.amd64 local proxy_dockerfile=Dockerfile.amd64 + local grafana_dockerfile=Dockerfile.amd64 # compat mode for SatoshiPortal repo # TODO: add more mappings? @@ -45,31 +57,23 @@ build_docker_images() { bitcoin_dockerfile="Dockerfile.arm32v6" clightning_dockerfile="Dockerfile.arm32v6" proxy_dockerfile="Dockerfile.arm32v6" + grafana_dockerfile="Dockerfile.arm32v6" fi trace "Creating cyphernodeconf image" - build_docker_image install/ cyphernode/cyphernodeconf:$CN_VERSION + build_docker_image install/ cyphernode/cyphernodeconf:$CONF_VERSION trace "Creating SatoshiPortal images" - build_docker_image install/SatoshiPortal/dockers/bitcoin-core cyphernode/bitcoin:$BC_VERSION $bitcoin_dockerfile - build_docker_image install/SatoshiPortal/dockers/c-lightning cyphernode/clightning:$CL_VERSION $clightning_dockerfile + build_docker_image install/SatoshiPortal/dockers/bitcoin-core cyphernode/bitcoin:$BITCOIN_VERSION $bitcoin_dockerfile + build_docker_image install/SatoshiPortal/dockers/c-lightning cyphernode/clightning:$LIGHTNING_VERSION $clightning_dockerfile trace "Creating cyphernode images" - build_docker_image api_auth_docker/ cyphernode/gatekeeper:$CN_VERSION - build_docker_image proxy_docker/ cyphernode/proxy:$CN_VERSION $proxy_dockerfile - build_docker_image cron_docker/ cyphernode/proxycron:$CN_VERSION - build_docker_image pycoin_docker/ cyphernode/pycoin:$CN_VERSION - build_docker_image otsclient_docker/ cyphernode/otsclient:$CN_VERSION + build_docker_image api_auth_docker/ cyphernode/gatekeeper:$GATEKEEPER_VERSION + build_docker_image proxy_docker/ cyphernode/proxy:$PROXY_VERSION $proxy_dockerfile + build_docker_image cron_docker/ cyphernode/proxycron:$PROXYCRON_VERSION + build_docker_image pycoin_docker/ cyphernode/pycoin:$PYCOIN_VERSION + build_docker_image otsclient_docker/ cyphernode/otsclient:$OTSCLIENT_VERSION } -# CYPHERNODE VERSION -GATEKEEPER_VERSION="latest" -PROXY_VERSION="latest" -PROXYCRON_VERSION="latest" -OTSCLIENT_VERSION="latest" -PYCOIN_VERSION="latest" -BITCOIN_VERSION="latest" -LIGHTNING_VERSION="latest" - build_docker_images diff --git a/dist/setup.sh b/dist/setup.sh index 94b25c4..708361a 100755 --- a/dist/setup.sh +++ b/dist/setup.sh @@ -183,6 +183,8 @@ configure() { # configure features of cyphernode docker run -v $current_path:/data \ -e DEFAULT_USER=$USER \ + -e DEFAULT_DATADIR_BASE=$HOME \ + -e SETUP_DIR=$SETUP_DIR \ -e DEFAULT_CERT_HOSTNAME=$(hostname) \ -e VERSION_OVERRIDE=$VERSION_OVERRIDE \ -e GATEKEEPER_VERSION=$GATEKEEPER_VERSION \ @@ -551,6 +553,11 @@ check_bitcoind() { echo 0 } +realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" +} + + sanity_checks() { echo " check requirements." @@ -649,6 +656,8 @@ PYCOIN_VERSION="v0.1" BITCOIN_VERSION="v0.17.0" LIGHTNING_VERSION="v0.6.2" +SETUP_DIR=$(dirname $(realpath $0)) + # trap ctrl-c and call ctrl_c() trap ctrl_c INT diff --git a/install/generator-cyphernode/generators/app/index.js b/install/generator-cyphernode/generators/app/index.js index 6053f7e..feca97b 100644 --- a/install/generator-cyphernode/generators/app/index.js +++ b/install/generator-cyphernode/generators/app/index.js @@ -124,6 +124,8 @@ module.exports = class extends Generator { } async _initConfig() { + this.defaultDataDirBase = process.env.DEFAULT_DATADIR_BASE; + this.setupDir = process.env.SETUP_DIR; const versionOverride = process.env.VERSION_OVERRIDE==='true'; if( fs.existsSync(this.destinationPath('config.7z')) ) { let r = {}; diff --git a/install/generator-cyphernode/generators/app/prompters/999_installer.js b/install/generator-cyphernode/generators/app/prompters/999_installer.js index a8e5ea7..24e04eb 100644 --- a/install/generator-cyphernode/generators/app/prompters/999_installer.js +++ b/install/generator-cyphernode/generators/app/prompters/999_installer.js @@ -37,16 +37,20 @@ module.exports = { default: utils._getDefault( 'gatekeeper_datapath' ), choices: [ { - name: "/var/run/cyphernode/gatekeeper (needs sudo and "+chalk.red('incompatible with OSX')+")", - value: "/var/run/cyphernode/gatekeeper" + name: utils.setupDir+"/cyphernode/gatekeeper", + value: utils.setupDir+"/cyphernode/gatekeeper" }, { - name: "~/.cyphernode/gatekeeper", - value: "~/.cyphernode/gatekeeper" + name: utils.defaultDataDirBase+"/cyphernode/gatekeeper", + value: utils.defaultDataDirBase+"/cyphernode/gatekeeper" }, { - name: "~/gatekeeper", - value: "~/gatekeeper" + name: utils.defaultDataDirBase+"/.cyphernode/gatekeeper", + value: utils.defaultDataDirBase+"/.cyphernode/gatekeeper" + }, + { + name: utils.defaultDataDirBase+"/gatekeeper", + value: utils.defaultDataDirBase+"/gatekeeper" }, { name: "Custom path", @@ -71,16 +75,20 @@ module.exports = { default: utils._getDefault( 'proxy_datapath' ), choices: [ { - name: "/var/run/cyphernode/proxy (needs sudo and "+chalk.red('incompatible with OSX')+")", - value: "/var/run/cyphernode/proxy" + name: utils.setupDir+"/cyphernode/proxy", + value: utils.setupDir+"/cyphernode/proxy" }, { - name: "~/.cyphernode/proxy", - value: "~/.cyphernode/proxy" + name: utils.defaultDataDirBase+"/cyphernode/proxy", + value: utils.defaultDataDirBase+"/cyphernode/proxy" }, { - name: "~/proxy", - value: "~/proxy" + name: utils.defaultDataDirBase+"/.cyphernode/proxy", + value: utils.defaultDataDirBase+"/.cyphernode/proxy" + }, + { + name: utils.defaultDataDirBase+"/proxy", + value: utils.defaultDataDirBase+"/proxy" }, { name: "Custom path", @@ -105,16 +113,20 @@ module.exports = { default: utils._getDefault( 'bitcoin_datapath' ), choices: [ { - name: "/var/run/cyphernode/bitcoin (needs sudo and "+chalk.red('incompatible with OSX')+")", - value: "/var/run/cyphernode/bitcoin" + name: utils.setupDir+"/cyphernode/bitcoin", + value: utils.setupDir+"/cyphernode/bitcoin" }, { - name: "~/.cyphernode/bitcoin", - value: "~/.cyphernode/bitcoin" + name: utils.defaultDataDirBase+"/cyphernode/bitcoin", + value: utils.defaultDataDirBase+"/cyphernode/bitcoin" }, { - name: "~/bitcoin", - value: "~/bitcoin" + name: utils.defaultDataDirBase+"/.cyphernode/bitcoin", + value: utils.defaultDataDirBase+"/.cyphernode/bitcoin" + }, + { + name: utils.defaultDataDirBase+"/bitcoin", + value: utils.defaultDataDirBase+"/bitcoin" }, { name: "Custom path", @@ -139,16 +151,20 @@ module.exports = { default: utils._getDefault( 'lightning_datapath' ), choices: [ { - name: "/var/run/cyphernode/lightning (needs sudo - "+chalk.red('incompatible with OSX')+")", - value: "/var/run/cyphernode/lightning" + name: utils.setupDir+"/cyphernode/lightning", + value: utils.setupDir+"/cyphernode/lightning" }, { - name: "~/.cyphernode/lightning", - value: "~/.cyphernode/lightning" + name: utils.defaultDataDirBase+"/cyphernode/lightning", + value: utils.defaultDataDirBase+"/cyphernode/lightning" }, { - name: "~/lightning", - value: "~/lightning" + name: utils.defaultDataDirBase+"/.cyphernode/lightning", + value: utils.defaultDataDirBase+"/.cyphernode/lightning" + }, + { + name: utils.defaultDataDirBase+"/lightning", + value: utils.defaultDataDirBase+"/lightning" }, { name: "Custom path", @@ -173,16 +189,20 @@ module.exports = { default: utils._getDefault( 'otsclient_datapath' ), choices: [ { - name: "/var/run/cyphernode/otsclient (needs sudo and "+chalk.red('incompatible with OSX')+")", - value: "/var/run/cyphernode/otsclient" + name: utils.setupDir+"/cyphernode/otsclient", + value: utils.setupDir+"/cyphernode/otsclient" }, { - name: "~/.cyphernode/otsclient", - value: "~/.cyphernode/otsclient" + name: utils.defaultDataDirBase+"/cyphernode/otsclient", + value: utils.defaultDataDirBase+"/cyphernode/otsclient" }, { - name: "~/otsclient", - value: "~/otsclient" + name: utils.defaultDataDirBase+"/.cyphernode/otsclient", + value: utils.defaultDataDirBase+"/.cyphernode/otsclient" + }, + { + name: utils.defaultDataDirBase+"/otsclient", + value: utils.defaultDataDirBase+"/otsclient" }, { name: "Custom path", From 49b779927182f8a6f51ed64adb99735e1368848e Mon Sep 17 00:00:00 2001 From: SKP Date: Sun, 20 Jan 2019 12:44:02 +0100 Subject: [PATCH 4/5] Check if var with current directory is empty to prevent infinite loop --- dist/setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dist/setup.sh b/dist/setup.sh index 708361a..b7b7bd5 100755 --- a/dist/setup.sh +++ b/dist/setup.sh @@ -528,6 +528,9 @@ check_directory_owner() { local status=0 for d in "${directories[@]}" do + if [[ ''$d == '' ]]; then + continue + fi if [[ -e $d ]]; then # is it mine and does it have rw ? # don't care about group rights From ed62ef409f25ac6c23663b0407fd687edbce69f2 Mon Sep 17 00:00:00 2001 From: SKP Date: Sun, 20 Jan 2019 13:00:04 +0100 Subject: [PATCH 5/5] using absolute paths in check_directory_owner() --- dist/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/setup.sh b/dist/setup.sh index b7b7bd5..592f1dd 100755 --- a/dist/setup.sh +++ b/dist/setup.sh @@ -531,6 +531,7 @@ check_directory_owner() { if [[ ''$d == '' ]]; then continue fi + d=$(realpath $d) if [[ -e $d ]]; then # is it mine and does it have rw ? # don't care about group rights