From ed71a2ed8fd09ea3a11fa7c997243e6e9a6b3c5d Mon Sep 17 00:00:00 2001 From: kexkey Date: Fri, 17 May 2019 18:29:06 -0400 Subject: [PATCH] First draft of the pub/sub notifier --- build.sh | 12 +-- dist/setup.sh | 19 +++-- .../generators/app/index.js | 2 + .../installer/docker/docker-compose.yaml | 23 ++++++ notifier_docker/Dockerfile | 17 +++++ notifier_docker/script/requesthandler.sh | 38 ++++++++++ notifier_docker/script/response.sh | 22 ++++++ notifier_docker/script/startnotifier.sh | 5 ++ notifier_docker/script/trace.sh | 15 ++++ notifier_docker/script/web.sh | 76 +++++++++++++++++++ proxy_docker/Dockerfile | 3 + proxy_docker/app/script/callbacks_job.sh | 9 ++- proxy_docker/app/script/ots.sh | 9 ++- 13 files changed, 231 insertions(+), 19 deletions(-) create mode 100644 notifier_docker/Dockerfile create mode 100644 notifier_docker/script/requesthandler.sh create mode 100644 notifier_docker/script/response.sh create mode 100644 notifier_docker/script/startnotifier.sh create mode 100644 notifier_docker/script/trace.sh create mode 100644 notifier_docker/script/web.sh diff --git a/build.sh b/build.sh index 59516e3..1532442 100755 --- a/build.sh +++ b/build.sh @@ -6,6 +6,7 @@ TRACING=1 CONF_VERSION="v0.2.0-local" GATEKEEPER_VERSION="v0.2.0-local" PROXY_VERSION="v0.2.0-local" +NOTIFIER_VERSION="v0.2.0-local" PROXYCRON_VERSION="v0.2.0-local" OTSCLIENT_VERSION="v0.2.0-local" PYCOIN_VERSION="v0.2.0-local" @@ -34,11 +35,12 @@ build_docker_images() { docker build install/ -t cyphernode/cyphernodeconf:$CONF_VERSION trace "Creating cyphernode images" - docker build api_auth_docker/ -t cyphernode/gatekeeper:$GATEKEEPER_VERSION - docker build proxy_docker/ -t cyphernode/proxy:$PROXY_VERSION - docker build cron_docker/ -t cyphernode/proxycron:$PROXYCRON_VERSION - docker build pycoin_docker/ -t cyphernode/pycoin:$PYCOIN_VERSION - docker build otsclient_docker/ -t cyphernode/otsclient:$OTSCLIENT_VERSION + docker build api_auth_docker/ -t cyphernode/gatekeeper:$GATEKEEPER_VERSION \ + && docker build proxy_docker/ -t cyphernode/proxy:$PROXY_VERSION \ + && docker build notifier_docker/ -t cyphernode/notifier:$NOTIFIER_VERSION \ + && docker build cron_docker/ -t cyphernode/proxycron:$PROXYCRON_VERSION \ + && docker build pycoin_docker/ -t cyphernode/pycoin:$PYCOIN_VERSION \ + && docker build otsclient_docker/ -t cyphernode/otsclient:$OTSCLIENT_VERSION } build_docker_images diff --git a/dist/setup.sh b/dist/setup.sh index f84e6b7..455bab7 100755 --- a/dist/setup.sh +++ b/dist/setup.sh @@ -189,6 +189,7 @@ configure() { -e VERSION_OVERRIDE=$VERSION_OVERRIDE \ -e GATEKEEPER_VERSION=$GATEKEEPER_VERSION \ -e PROXY_VERSION=$PROXY_VERSION \ + -e NOTIFIER_VERSION=$NOTIFIER_VERSION \ -e PROXYCRON_VERSION=$PROXYCRON_VERSION \ -e OTSCLIENT_VERSION=$OTSCLIENT_VERSION \ -e PYCOIN_VERSION=$PYCOIN_VERSION \ @@ -711,14 +712,15 @@ AUTOSTART=0 # CYPHERNODE VERSION "v0.1.1" VERSION_OVERRIDE="true" -CONF_VERSION="v0.2.0-test" -GATEKEEPER_VERSION="v0.2.0-test" -PROXY_VERSION="v0.2.0-test" -PROXYCRON_VERSION="v0.2.0-test" -OTSCLIENT_VERSION="v0.2.0-test" -PYCOIN_VERSION="v0.2.0-test" -BITCOIN_VERSION="v0.17.1-test" -LIGHTNING_VERSION="v0.7.0-test" +CONF_VERSION="v0.2.0" +GATEKEEPER_VERSION="v0.2.0" +PROXY_VERSION="v0.2.0" +NOTIFIER_VERSION="v0.2.0" +PROXYCRON_VERSION="v0.2.0" +OTSCLIENT_VERSION="v0.2.0" +PYCOIN_VERSION="v0.2.0" +BITCOIN_VERSION="v0.17.1" +LIGHTNING_VERSION="v0.7.0" SETUP_DIR=$(dirname $(realpath $0)) @@ -796,6 +798,7 @@ if [[ $nbbuiltimgs -gt 1 ]]; then CONF_VERSION="$CONF_VERSION-local" GATEKEEPER_VERSION="$GATEKEEPER_VERSION-local" PROXY_VERSION="$PROXY_VERSION-local" + NOTIFIER_VERSION="$NOTIFIER_VERSION-local" PROXYCRON_VERSION="$PROXYCRON_VERSION-local" OTSCLIENT_VERSION="$OTSCLIENT_VERSION-local" PYCOIN_VERSION="$PYCOIN_VERSION-local" diff --git a/install/generator-cyphernode/generators/app/index.js b/install/generator-cyphernode/generators/app/index.js index aafc2c8..5a83b6a 100644 --- a/install/generator-cyphernode/generators/app/index.js +++ b/install/generator-cyphernode/generators/app/index.js @@ -225,6 +225,7 @@ module.exports = class extends Generator { if( versionOverride ) { delete this.props.gatekeeper_version; delete this.props.proxy_version; + delete this.props.notifier_version; delete this.props.proxycron_version; delete this.props.pycoin_version; delete this.props.otsclient_version; @@ -464,6 +465,7 @@ module.exports = class extends Generator { default_username: process.env.DEFAULT_USER || '', gatekeeper_version: process.env.GATEKEEPER_VERSION || 'latest', proxy_version: process.env.PROXY_VERSION || 'latest', + notifier_version: process.env.NOTIFIER_VERSION || 'latest', proxycron_version: process.env.PROXYCRON_VERSION || 'latest', pycoin_version: process.env.PYCOIN_VERSION || 'latest', otsclient_version: process.env.OTSCLIENT_VERSION || 'latest', diff --git a/install/generator-cyphernode/generators/app/templates/installer/docker/docker-compose.yaml b/install/generator-cyphernode/generators/app/templates/installer/docker/docker-compose.yaml index 6776c45..b6a9c64 100644 --- a/install/generator-cyphernode/generators/app/templates/installer/docker/docker-compose.yaml +++ b/install/generator-cyphernode/generators/app/templates/installer/docker/docker-compose.yaml @@ -167,6 +167,29 @@ services: restart: always <% } %> + broker: + image: eclipse-mosquitto:1.6 +# deploy: +# placement: +# constraints: [node.hostname==dev] +# ports: +# - "1883:1883" +# - "9001:9001" + networks: + - cyphernodenet + restart: always + + notifier: + image: cyphernode/notifier:<%= notifier_version %> + command: $USER ./startnotifier.sh +# deploy: +# placement: +# constraints: [node.hostname==dev] + networks: + - cyphernodenet + - cyphernodeappsnet + restart: always + networks: cyphernodenet: external: true diff --git a/notifier_docker/Dockerfile b/notifier_docker/Dockerfile new file mode 100644 index 0000000..3da0f2b --- /dev/null +++ b/notifier_docker/Dockerfile @@ -0,0 +1,17 @@ +FROM eclipse-mosquitto:1.6 + +ENV HOME /notifier + +RUN apk --no-cache --update add jq curl su-exec + +WORKDIR ${HOME} + +COPY script/* ./ + +RUN chmod +x startnotifier.sh requesthandler.sh \ + && chmod o+w . + +ENTRYPOINT ["su-exec"] + +# docker run --rm -d -p 1883:1883 -p 9001:9001 --network cyphernodenet --name broker eclipse-mosquitto +# docker run --rm -it --network cyphernodenet --name mq1 mqtt-client diff --git a/notifier_docker/script/requesthandler.sh b/notifier_docker/script/requesthandler.sh new file mode 100644 index 0000000..51a14fe --- /dev/null +++ b/notifier_docker/script/requesthandler.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +. ./trace.sh +. ./web.sh +. ./response.sh + +main() { + trace "Entering main()..." + + local msg + local cmd + local response + local response_topic + + while read msg; do + trace "[main] msg=${msg}" + + cmd=$(echo ${msg} | jq ".cmd" | tr -d '"') + trace "[main] cmd=${cmd}" + + response_topic=$(echo ${msg} | jq '."response-topic"' | tr -d '"') + trace "[main] response_topic=${response_topic}" + + case "${cmd}" in + web) + response=$(web "${msg}") + publish_response "${response}" "${response_topic}" ${?} + trace "[main] PR" + ;; + esac + trace "[main] case finished" + done +} + +export TRACING=1 + +main +exit $? diff --git a/notifier_docker/script/response.sh b/notifier_docker/script/response.sh new file mode 100644 index 0000000..288687d --- /dev/null +++ b/notifier_docker/script/response.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +. ./trace.sh + +publish_response() { + trace "Entering publish_response()..." + + local response=${1} + local response_topic=${2} + local returncode=${3} + + trace "[publish_response] response=${response}" + trace "[publish_response] response_topic=${response_topic}" + trace "[publish_response] returncode=${returncode}" + + trace "[publish_response] mosquitto_pub -h broker -t \"${response_topic}\" -m \"${response}\"" + mosquitto_pub -h broker -t "${response_topic}" -m "${response}" + returncode=$? + trace_rc ${returncode} + + return ${returncode} +} diff --git a/notifier_docker/script/startnotifier.sh b/notifier_docker/script/startnotifier.sh new file mode 100644 index 0000000..64bc336 --- /dev/null +++ b/notifier_docker/script/startnotifier.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +. ./trace.sh + +mosquitto_sub -h broker -t notifier | ./requesthandler.sh diff --git a/notifier_docker/script/trace.sh b/notifier_docker/script/trace.sh new file mode 100644 index 0000000..4c0a1c2 --- /dev/null +++ b/notifier_docker/script/trace.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +trace() +{ + if [ -n "${TRACING}" ]; then + echo "$(date -Is) $$ ${1}" 1>&2 + fi +} + +trace_rc() +{ + if [ -n "${TRACING}" ]; then + echo "$(date -Is) $$ Last return code: ${1}" 1>&2 + fi +} diff --git a/notifier_docker/script/web.sh b/notifier_docker/script/web.sh new file mode 100644 index 0000000..f35d39f --- /dev/null +++ b/notifier_docker/script/web.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +. ./trace.sh + +web() { + trace "Entering web()..." + + local msg=${1} + local url + local body + local returncode + local http_code + local result + + trace "[web] msg=${msg}" + url=$(echo ${msg} | jq ".url") + trace "[web] url=${url}" + + body=$(echo ${msg} | jq -e ".body") + # jq -e will have a return code of 1 if the supplied tag is null. + if [ "$?" -eq "0" ]; then + # body tag not null, so it's a POST + trace "[web] body=${body}" + else + body= + trace "[web] no body, GET request" + fi + + http_code=$(curl_it "${url}" "${body}") + returncode=$? + trace_rc ${returncode} + + if [ "${returncode}" -eq "0" ]; then + # {"result":"success", "response":""} + result="success" + else + # {"result":"error", "response":""} + result="error" + fi + + echo "{\"result\":\"${result}\",\"http_code\":\"${http_code}\"}" + + return ${returncode} +} + +curl_it() { + trace "Entering curl_it()..." + + local url=$(echo "${1}" | tr -d '"') + local data=${2} + local returncode + + if [ -n "${data}" ]; then + trace "[curl_it] curl -o /dev/null -w \"%{http_code}\" -H \"Content-Type: application/json\" -H \"X-Forwarded-Proto: https\" -d ${data} ${url}" + rc=$(curl -o /dev/null -w "%{http_code}" -H "Content-Type: application/json" -H "X-Forwarded-Proto: https" -d ${data} ${url}) + returncode=$? + else + trace "[curl_it] curl -o /dev/null -w \"%{http_code}\" ${url}" + rc=$(curl -o /dev/null -w "%{http_code}" ${url}) + returncode=$? + fi + trace "[curl_it] HTTP return code=${rc}" + trace_rc ${returncode} + + echo "${rc}" + + if [ "${returncode}" -eq "0" ]; then + if [ "${rc}" -lt "400" ]; then + return 0 + else + return ${rc} + fi + else + return ${returncode} + fi +} diff --git a/proxy_docker/Dockerfile b/proxy_docker/Dockerfile index fad4969..de7a270 100644 --- a/proxy_docker/Dockerfile +++ b/proxy_docker/Dockerfile @@ -13,6 +13,9 @@ WORKDIR ${HOME} COPY app/data/* ./ COPY app/script/* ./ COPY --from=cyphernode/clightning:v0.7.0-test /usr/local/bin/lightning-cli ./ +# COPY --from=eclipse-mosquitto:1.6 /usr/bin/mosquitto_sub ./ +# COPY --from=eclipse-mosquitto:1.6 /usr/bin/mosquitto_pub ./ +COPY --from=eclipse-mosquitto:1.6 /usr/bin/mosquitto_rr ./ RUN chmod +x startproxy.sh requesthandler.sh lightning-cli sqlmigrate*.sh waitanyinvoice.sh \ && chmod o+w . \ diff --git a/proxy_docker/app/script/callbacks_job.sh b/proxy_docker/app/script/callbacks_job.sh index 88425cd..ec3b761 100644 --- a/proxy_docker/app/script/callbacks_job.sh +++ b/proxy_docker/app/script/callbacks_job.sh @@ -237,9 +237,12 @@ curl_callback() { local data=${2} local returncode - trace "[curl_callback] curl -w \"%{http_code}\" -H \"Content-Type: application/json\" -H \"X-Forwarded-Proto: https\" -d \"${data}\" ${url}" - rc=$(curl -w "%{http_code}" -H "Content-Type: application/json" -H "X-Forwarded-Proto: https" -d "${data}" ${url}) - returncode=$? + #trace "[curl_callback] curl -w \"%{http_code}\" -H \"Content-Type: application/json\" -H \"X-Forwarded-Proto: https\" -d \"${data}\" ${url}" + #rc=$(curl -w "%{http_code}" -H "Content-Type: application/json" -H "X-Forwarded-Proto: https" -d "${data}" ${url}) + #returncode=$? + trace "[curl_callback] mosquitto_rr -h broker -t notifier -e jefsio -m \"{\"response-topic\":\"jefsio\",\"cmd\":\"web\",\"url\":\"${url}\",\"body\":\"${data}\"}\"" + rc=$(./mosquitto_rr -h broker -t notifier -e jefsio -m "{\"response-topic\":\"jefsio\",\"cmd\":\"web\",\"url\":\"${url}\",\"body\":\"${data}\"}") + rc=$(echo "${rc}" | jq ".http_code") trace "[curl_callback] HTTP return code=${rc}" trace_rc ${returncode} diff --git a/proxy_docker/app/script/ots.sh b/proxy_docker/app/script/ots.sh index e1fc41c..42d9474 100644 --- a/proxy_docker/app/script/ots.sh +++ b/proxy_docker/app/script/ots.sh @@ -202,9 +202,12 @@ serve_ots_backoffice() { trace "[serve_ots_backoffice] url=${url}" # Call back newly upgraded stamps - trace "[serve_ots_backoffice] curl -s -o /dev/null -w \"%{http_code}\" -H \"X-Forwarded-Proto: https\" ${url}" - rc=$(curl -s -o /dev/null -w "%{http_code}" -H "X-Forwarded-Proto: https" ${url}) - returncode=$? + #trace "[serve_ots_backoffice] curl -s -o /dev/null -w \"%{http_code}\" -H \"X-Forwarded-Proto: https\" ${url}" + #rc=$(curl -s -o /dev/null -w "%{http_code}" -H "X-Forwarded-Proto: https" ${url}) + #returncode=$? + trace "[serve_ots_backoffice] mosquitto_rr -h broker -t notifier -e dhtsggs -m \"{\"response-topic\":\"dhtsggs\",\"cmd\":\"web\",\"url\":\"${url}\"}\"" + rc=$(./mosquitto_rr -h broker -t notifier -e dhtsggs -m "{\"response-topic\":\"dhtsggs\",\"cmd\":\"web\",\"url\":\"${url}\"}") + rc=$(echo "${rc}" | jq ".http_code") trace_rc ${returncode} # Even if curl executed ok, we need to make sure the http return code is also ok