From 33de124a521299a5d9a76655f47f183cb345b7f7 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Wed, 28 Nov 2018 18:47:34 +0900 Subject: [PATCH 01/36] fix doge --- docker-compose-generator/docker-fragments/dogecoin.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-generator/docker-fragments/dogecoin.yml b/docker-compose-generator/docker-fragments/dogecoin.yml index b5777bf..57642ec 100644 --- a/docker-compose-generator/docker-fragments/dogecoin.yml +++ b/docker-compose-generator/docker-fragments/dogecoin.yml @@ -27,8 +27,8 @@ services: nbxplorer: environment: NBXPLORER_CHAINS: "doge" - NBXPLORER_VIARPCURL: http://dogecoind:22555/ - NBXPLORER_VIANODEENDPOINT: dogecoind:22556 + NBXPLORER_DOGERPCURL: http://dogecoind:22555/ + NBXPLORER_DOGENODEENDPOINT: dogecoind:22556 NBXPLORER_DOGERPCUSER: ceiwHEbqWI83 NBXPLORER_DOGERPCPASSWORD: DwubwWsoo3 links: From 244ce99f830d217854c1f9268d3b7bbfba2d5313 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Thu, 29 Nov 2018 19:46:33 +0900 Subject: [PATCH 02/36] Add scripts to upload and download utxosets --- contrib/load-utxo-set.sh | 43 +++++++++++++++++++++ contrib/save-utxo-set.sh | 82 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 contrib/load-utxo-set.sh create mode 100644 contrib/save-utxo-set.sh diff --git a/contrib/load-utxo-set.sh b/contrib/load-utxo-set.sh new file mode 100644 index 0000000..cd15df5 --- /dev/null +++ b/contrib/load-utxo-set.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# This script shows the steps to download and update an archive of the current UTXO Set + +exit # This script is not meant to run automatically + + +## ARGS# +NETWORK="testnet" +DOWNLOAD_LINK="http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-testnet-1445586.tar" +TAR_FILE_HASH="eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742" +####### + +BITCOIN_DATA_DIR="/var/lib/docker/volumes/generated_bitcoin_datadir/_data" +[ ! -d "$BITCOIN_DATA_DIR" ] && mkdir -p "$BITCOIN_DATA_DIR" + +TAR_FILE="$BITCOIN_DATA_DIR/snapshot.tar" +echo "Downloading $DOWNLOAD_LINK to $TAR_FILE" +wget "$DOWNLOAD_LINK" -q --show-progress -O "$TAR_FILE" + +if ! echo "$TAR_FILE_HASH" "$TAR_FILE" | sha256sum -c -; then + echo "Invalid hash" + exit 1 +fi + +NETWORK_DIRECTORY=$NETWORK +if [[ $NETWORK == "mainnet" ]]; then + NETWORK_DIRECTORY="." +fi +if [[ $NETWORK == "testnet" ]]; then + NETWORK_DIRECTORY="testnet3" +fi + +NETWORK_DIRECTORY="$BITCOIN_DATA_DIR/$NETWORK_DIRECTORY" +[ -d "$NETWORK_DIRECTORY/blocks" ] && rm -rf "$NETWORK_DIRECTORY/blocks" +[ -d "$NETWORK_DIRECTORY/chainstate" ] && rm -rf "$NETWORK_DIRECTORY/chainstate" +[ ! -d "$NETWORK_DIRECTORY" ] && mkdir "$NETWORK_DIRECTORY" + +echo "Extracting..." +tar -xf "$TAR_FILE" -C "$BITCOIN_DATA_DIR" + +echo "Extracted" +rm "$TAR_FILE" \ No newline at end of file diff --git a/contrib/save-utxo-set.sh b/contrib/save-utxo-set.sh new file mode 100644 index 0000000..42baac9 --- /dev/null +++ b/contrib/save-utxo-set.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# This script shows the steps to create an archive of the current UTXO Set + +exit # This script is not meant to run automatically + + +## ARGS# +NETWORK="testnet" +export AZURE_STORAGE_CONTAINER="public" +export AZURE_STORAGE_CONNECTION_STRING="" +####### + +# IN THE HOST ############################################################# + +# Stop btcpay +btcpay-down.sh + +# Run only bitcoind and connect to it +cd "`dirname $BTCPAY_ENV_FILE`" +docker-compose -f $BTCPAY_DOCKER_COMPOSE run -e "NETWORK=$NETWORK" bitcoind bash + +# IN THE CONTAINER ############################################################# +ENVIRONMENT="" +BITCOIND="bitcoind -datadir=/data" +BITCOIN_CLI="bitcoin-cli -datadir=/data" + +$BITCOIND & +BITCOIND_PID=$! +CURRENT_HEIGHT="$($BITCOIN_CLI -rpcwait getblockcount)" +let "PRUNED_HEIGHT=$CURRENT_HEIGHT - 289" + +echo "Pruning to $PRUNED_HEIGHT" +$BITCOIN_CLI pruneblockchain "$PRUNED_HEIGHT" + +echo "Waiting bitcoind to stop..." +$BITCOIN_CLI stop +wait $BITCOIND_PID + +NETWORK_DIRECTORY=$NETWORK +if [[ $NETWORK == "mainnet" ]]; then + NETWORK_DIRECTORY="." +fi +if [[ $NETWORK == "testnet" ]]; then + NETWORK_DIRECTORY="testnet3" +fi + +cd /data +TAR_NAME="utxo-snapshot-bitcoin-$NETWORK-$PRUNED_HEIGHT.tar" +echo "Creating $TAR_NAME..." +tar -cf "$TAR_NAME" "$NETWORK_DIRECTORY/blocks/" +tar -rf "$TAR_NAME" "$NETWORK_DIRECTORY/chainstate/" + +# Exit from the container +exit + +# IN THE HOST ############################################################# + +# Restart btcpay +btcpay-up.sh + +TAR_FILE="$(echo /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*)" +TAR_FILE_HASH="$(sha256sum "$TAR_FILE" | cut -d " " -f 1)" +echo "SHA256: $TAR_FILE_HASH" +echo "Uploading to azure..." +# Install az from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest + +BLOB_NAME="$(basename -- $TAR_FILE)" +az storage container create --name "$AZURE_STORAGE_CONTAINER" --public-access "blob" +az storage blob upload -f "$TAR_FILE" \ + -c "$AZURE_STORAGE_CONTAINER" \ + -n "$BLOB_NAME" \ + --content-type "application/x-tar" + +az storage blob metadata update --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --metadata "sha256=$TAR_FILE_HASH" + +# Print the sha256sum. Downloaders will need to verify this +STORAGE_URL="$(az storage blob url --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --protocol "http")" +echo "You can now download the UTXO on $STORAGE_URL" +echo "Please, after download, verify the sha256 with:" +echo "echo "$TAR_FILE_HASH $BLOB_NAME" | sha256sum -c -" +rm "$TAR_FILE" \ No newline at end of file From 438b1046af1870ab0e27bfb1a226162a3b585e8e Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Thu, 29 Nov 2018 19:58:56 +0900 Subject: [PATCH 03/36] Allow dogecoin pruning --- .../docker-fragments/opt-save-storage-s.yml | 5 ++++- .../docker-fragments/opt-save-storage-xs.yml | 5 ++++- .../docker-fragments/opt-save-storage-xxs.yml | 5 ++++- .../docker-fragments/opt-save-storage.yml | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-s.yml b/docker-compose-generator/docker-fragments/opt-save-storage-s.yml index e134222..9acf2c0 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-s.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-s.yml @@ -20,4 +20,7 @@ services: BITCOIN_EXTRA_ARGS: prune=50000 viacoind: environment: - BITCOIN_EXTRA_ARGS: prune=50000 \ No newline at end of file + BITCOIN_EXTRA_ARGS: prune=50000 + dogecoind: + environment: + DOGECOIN_EXTRA_ARGS: prune=50000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml b/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml index 8e32c8f..7b95f20 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml @@ -20,4 +20,7 @@ services: BITCOIN_EXTRA_ARGS: prune=25000 viacoind: environment: - BITCOIN_EXTRA_ARGS: prune=25000 \ No newline at end of file + BITCOIN_EXTRA_ARGS: prune=25000 + dogecoind: + environment: + DOGECOIN_EXTRA_ARGS: prune=25000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml b/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml index 6ad1d81..4e8ce5f 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml @@ -20,4 +20,7 @@ services: BITCOIN_EXTRA_ARGS: prune=5000 viacoind: environment: - BITCOIN_EXTRA_ARGS: prune=5000 \ No newline at end of file + BITCOIN_EXTRA_ARGS: prune=5000 + dogecoind: + environment: + DOGECOIN_EXTRA_ARGS: prune=5000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage.yml b/docker-compose-generator/docker-fragments/opt-save-storage.yml index 9e440cd..4a88674 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage.yml @@ -20,4 +20,7 @@ services: BITCOIN_EXTRA_ARGS: prune=100000 viacoind: environment: - BITCOIN_EXTRA_ARGS: prune=100000 \ No newline at end of file + BITCOIN_EXTRA_ARGS: prune=100000 + dogecoind: + environment: + DOGECOIN_EXTRA_ARGS: prune=100000 \ No newline at end of file From 9f7d768fab292f2e56d14b1384c9c993c36b57fe Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 00:07:49 +0900 Subject: [PATCH 04/36] Improve save/load utxo --- contrib/load-utxo-set.sh | 60 +++++++++++++++++++++------- contrib/save-utxo-set-in-bitcoind.sh | 28 +++++++++++++ contrib/save-utxo-set.sh | 45 ++------------------- 3 files changed, 78 insertions(+), 55 deletions(-) create mode 100644 contrib/save-utxo-set-in-bitcoind.sh diff --git a/contrib/load-utxo-set.sh b/contrib/load-utxo-set.sh index cd15df5..8911311 100644 --- a/contrib/load-utxo-set.sh +++ b/contrib/load-utxo-set.sh @@ -2,26 +2,51 @@ # This script shows the steps to download and update an archive of the current UTXO Set -exit # This script is not meant to run automatically +if ! [ "$0" = "$BASH_SOURCE" ]; then + echo "This script must not be sourced" + exit 1 +fi +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root after running \"sudo su -\"" + exit 1 +fi -## ARGS# -NETWORK="testnet" -DOWNLOAD_LINK="http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-testnet-1445586.tar" -TAR_FILE_HASH="eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742" -####### +: "${UTXO_DOWNLOAD_LINK:=http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-mainnet-551636.tar}" +NETWORK="" +[[ $DOWNLOAD_LINK == *-testnet-* ]] && NETWORK="testnet" +[[ $DOWNLOAD_LINK == *-mainnet-* ]] && NETWORK="mainnet" +[[ $DOWNLOAD_LINK == *-regtest-* ]] && NETWORK="regtest" BITCOIN_DATA_DIR="/var/lib/docker/volumes/generated_bitcoin_datadir/_data" [ ! -d "$BITCOIN_DATA_DIR" ] && mkdir -p "$BITCOIN_DATA_DIR" -TAR_FILE="$BITCOIN_DATA_DIR/snapshot.tar" -echo "Downloading $DOWNLOAD_LINK to $TAR_FILE" -wget "$DOWNLOAD_LINK" -q --show-progress -O "$TAR_FILE" +TAR_NAME="$(basename $DOWNLOAD_LINK)" +TAR_FILE="$BITCOIN_DATA_DIR/$TAR_NAME" -if ! echo "$TAR_FILE_HASH" "$TAR_FILE" | sha256sum -c -; then - echo "Invalid hash" +echo "Downloading $DOWNLOAD_LINK to $TAR_FILE" +cd "$BITCOIN_DATA_DIR" +if [ ! -f "$TAR_FILE" ]; then + wget "$DOWNLOAD_LINK" -q --show-progress +else + echo "$TAR_FILE already exists" +fi + +echo " +fab994299273080bf7124c8c45c4ada867974ca747900178496a69e450cf713f utxo-snapshot-bitcoin-mainnet-551636.tar +eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742 utxo-snapshot-bitcoin-testnet-1445586.tar +" > "trusted-utxo-sets.asc" + +grep "$TAR_NAME" "trusted-utxo-sets.asc" | tee "sig.asc" +rm "trusted-utxo-sets.asc" +if ! sha256sum -c "sig.asc"; then + echo "$TAR_FILE is not trusted" + rm "sig.asc" + cd - exit 1 fi +rm "sig.asc" +cd - NETWORK_DIRECTORY=$NETWORK if [[ $NETWORK == "mainnet" ]]; then @@ -37,7 +62,14 @@ NETWORK_DIRECTORY="$BITCOIN_DATA_DIR/$NETWORK_DIRECTORY" [ ! -d "$NETWORK_DIRECTORY" ] && mkdir "$NETWORK_DIRECTORY" echo "Extracting..." -tar -xf "$TAR_FILE" -C "$BITCOIN_DATA_DIR" +if ! tar -xf "$TAR_FILE" -C "$BITCOIN_DATA_DIR"; then + echo "Failed extracting, did you turned bitcoin off? (btcpay-down.sh)" + exit 1 +fi +rm "$TAR_FILE" -echo "Extracted" -rm "$TAR_FILE" \ No newline at end of file +BTCPAY_DATA_DIR="/var/lib/docker/volumes/generated_btcpay_datadir/_data" +[ ! -d "$BTCPAY_DATA_DIR" ] && mkdir -p "$BTCPAY_DATA_DIR" +echo "$TAR_NAME" > "$BTCPAY_DATA_DIR/FastSynced" + +echo "Successfully downloaded and extracted, you can run btcpay again (btcpay-up.sh)" \ No newline at end of file diff --git a/contrib/save-utxo-set-in-bitcoind.sh b/contrib/save-utxo-set-in-bitcoind.sh new file mode 100644 index 0000000..7f4dde5 --- /dev/null +++ b/contrib/save-utxo-set-in-bitcoind.sh @@ -0,0 +1,28 @@ +BITCOIND="bitcoind -datadir=/data" +BITCOIN_CLI="bitcoin-cli -datadir=/data" + +$BITCOIND & +BITCOIND_PID=$! +CURRENT_HEIGHT="$($BITCOIN_CLI -rpcwait getblockcount)" +let "PRUNED_HEIGHT=$CURRENT_HEIGHT - 289" + +echo "Pruning to $PRUNED_HEIGHT" +$BITCOIN_CLI pruneblockchain "$PRUNED_HEIGHT" + +echo "Waiting bitcoind to stop..." +$BITCOIN_CLI stop +wait $BITCOIND_PID + +NETWORK_DIRECTORY=$NETWORK +if [[ $NETWORK == "mainnet" ]]; then + NETWORK_DIRECTORY="." +fi +if [[ $NETWORK == "testnet" ]]; then + NETWORK_DIRECTORY="testnet3" +fi + +cd /data +TAR_NAME="utxo-snapshot-bitcoin-$NETWORK-$PRUNED_HEIGHT.tar" +echo "Creating $TAR_NAME..." +tar -cf "$TAR_NAME" "$NETWORK_DIRECTORY/blocks/" +tar -rf "$TAR_NAME" "$NETWORK_DIRECTORY/chainstate/" diff --git a/contrib/save-utxo-set.sh b/contrib/save-utxo-set.sh index 42baac9..aeee683 100644 --- a/contrib/save-utxo-set.sh +++ b/contrib/save-utxo-set.sh @@ -2,11 +2,11 @@ # This script shows the steps to create an archive of the current UTXO Set -exit # This script is not meant to run automatically +exit 1 # This script is not meant to run automatically ## ARGS# -NETWORK="testnet" +NETWORK="mainnet" export AZURE_STORAGE_CONTAINER="public" export AZURE_STORAGE_CONNECTION_STRING="" ####### @@ -17,46 +17,9 @@ export AZURE_STORAGE_CONNECTION_STRING="" btcpay-down.sh # Run only bitcoind and connect to it +SCRIPT="$(cat save-utxo-set-in-bitcoind.sh)" cd "`dirname $BTCPAY_ENV_FILE`" -docker-compose -f $BTCPAY_DOCKER_COMPOSE run -e "NETWORK=$NETWORK" bitcoind bash - -# IN THE CONTAINER ############################################################# -ENVIRONMENT="" -BITCOIND="bitcoind -datadir=/data" -BITCOIN_CLI="bitcoin-cli -datadir=/data" - -$BITCOIND & -BITCOIND_PID=$! -CURRENT_HEIGHT="$($BITCOIN_CLI -rpcwait getblockcount)" -let "PRUNED_HEIGHT=$CURRENT_HEIGHT - 289" - -echo "Pruning to $PRUNED_HEIGHT" -$BITCOIN_CLI pruneblockchain "$PRUNED_HEIGHT" - -echo "Waiting bitcoind to stop..." -$BITCOIN_CLI stop -wait $BITCOIND_PID - -NETWORK_DIRECTORY=$NETWORK -if [[ $NETWORK == "mainnet" ]]; then - NETWORK_DIRECTORY="." -fi -if [[ $NETWORK == "testnet" ]]; then - NETWORK_DIRECTORY="testnet3" -fi - -cd /data -TAR_NAME="utxo-snapshot-bitcoin-$NETWORK-$PRUNED_HEIGHT.tar" -echo "Creating $TAR_NAME..." -tar -cf "$TAR_NAME" "$NETWORK_DIRECTORY/blocks/" -tar -rf "$TAR_NAME" "$NETWORK_DIRECTORY/chainstate/" - -# Exit from the container -exit - -# IN THE HOST ############################################################# - -# Restart btcpay +docker-compose -f $BTCPAY_DOCKER_COMPOSE run -e "NETWORK=$NETWORK" bitcoind bash -c "$SCRIPT" btcpay-up.sh TAR_FILE="$(echo /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*)" From c1f065ec2e485c065976799ededbb611116b976e Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 00:08:46 +0900 Subject: [PATCH 05/36] fix chmod --- contrib/load-utxo-set.sh | 0 contrib/save-utxo-set-in-bitcoind.sh | 0 contrib/save-utxo-set.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 contrib/load-utxo-set.sh mode change 100644 => 100755 contrib/save-utxo-set-in-bitcoind.sh mode change 100644 => 100755 contrib/save-utxo-set.sh diff --git a/contrib/load-utxo-set.sh b/contrib/load-utxo-set.sh old mode 100644 new mode 100755 diff --git a/contrib/save-utxo-set-in-bitcoind.sh b/contrib/save-utxo-set-in-bitcoind.sh old mode 100644 new mode 100755 diff --git a/contrib/save-utxo-set.sh b/contrib/save-utxo-set.sh old mode 100644 new mode 100755 From bcc8d8e965dbdab4a9f5a20e8588cc37277ed4fe Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 00:22:20 +0900 Subject: [PATCH 06/36] Do not upload to azure if not configure --- contrib/save-utxo-set-in-bitcoind.sh | 8 ++-- contrib/save-utxo-set.sh | 55 +++++++++++++++------------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/contrib/save-utxo-set-in-bitcoind.sh b/contrib/save-utxo-set-in-bitcoind.sh index 7f4dde5..6ebfd73 100755 --- a/contrib/save-utxo-set-in-bitcoind.sh +++ b/contrib/save-utxo-set-in-bitcoind.sh @@ -13,16 +13,16 @@ echo "Waiting bitcoind to stop..." $BITCOIN_CLI stop wait $BITCOIND_PID -NETWORK_DIRECTORY=$NETWORK -if [[ $NETWORK == "mainnet" ]]; then +NETWORK_DIRECTORY=$NBITCOIN_NETWORK +if [[ $NBITCOIN_NETWORK == "mainnet" ]]; then NETWORK_DIRECTORY="." fi -if [[ $NETWORK == "testnet" ]]; then +if [[ $NBITCOIN_NETWORK == "testnet" ]]; then NETWORK_DIRECTORY="testnet3" fi cd /data -TAR_NAME="utxo-snapshot-bitcoin-$NETWORK-$PRUNED_HEIGHT.tar" +TAR_NAME="utxo-snapshot-bitcoin-$NBITCOIN_NETWORK-$PRUNED_HEIGHT.tar" echo "Creating $TAR_NAME..." tar -cf "$TAR_NAME" "$NETWORK_DIRECTORY/blocks/" tar -rf "$TAR_NAME" "$NETWORK_DIRECTORY/chainstate/" diff --git a/contrib/save-utxo-set.sh b/contrib/save-utxo-set.sh index aeee683..b713c94 100755 --- a/contrib/save-utxo-set.sh +++ b/contrib/save-utxo-set.sh @@ -2,44 +2,49 @@ # This script shows the steps to create an archive of the current UTXO Set -exit 1 # This script is not meant to run automatically - - -## ARGS# -NETWORK="mainnet" -export AZURE_STORAGE_CONTAINER="public" -export AZURE_STORAGE_CONNECTION_STRING="" -####### +: "${AZURE_STORAGE_CONTAINER:=public}" # IN THE HOST ############################################################# # Stop btcpay btcpay-down.sh +for i in /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*; do + echo "Deleting $i" + rm $i +done + +rm /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-* # Run only bitcoind and connect to it SCRIPT="$(cat save-utxo-set-in-bitcoind.sh)" cd "`dirname $BTCPAY_ENV_FILE`" -docker-compose -f $BTCPAY_DOCKER_COMPOSE run -e "NETWORK=$NETWORK" bitcoind bash -c "$SCRIPT" +docker-compose -f $BTCPAY_DOCKER_COMPOSE run -e "NBITCOIN_NETWORK=$NBITCOIN_NETWORK" bitcoind bash -c "$SCRIPT" btcpay-up.sh +echo "Calculating the hash of the tar file..." TAR_FILE="$(echo /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*)" TAR_FILE_HASH="$(sha256sum "$TAR_FILE" | cut -d " " -f 1)" -echo "SHA256: $TAR_FILE_HASH" -echo "Uploading to azure..." -# Install az from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest -BLOB_NAME="$(basename -- $TAR_FILE)" -az storage container create --name "$AZURE_STORAGE_CONTAINER" --public-access "blob" -az storage blob upload -f "$TAR_FILE" \ - -c "$AZURE_STORAGE_CONTAINER" \ - -n "$BLOB_NAME" \ - --content-type "application/x-tar" +if [[ "$AZURE_STORAGE_CONNECTION_STRING" ]]; then + echo "Uploading to azure..." + # Install az from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest -az storage blob metadata update --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --metadata "sha256=$TAR_FILE_HASH" + BLOB_NAME="$(basename -- $TAR_FILE)" + az storage container create --name "$AZURE_STORAGE_CONTAINER" --public-access "blob" + az storage blob upload -f "$TAR_FILE" \ + -c "$AZURE_STORAGE_CONTAINER" \ + -n "$BLOB_NAME" \ + --content-type "application/x-tar" -# Print the sha256sum. Downloaders will need to verify this -STORAGE_URL="$(az storage blob url --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --protocol "http")" -echo "You can now download the UTXO on $STORAGE_URL" -echo "Please, after download, verify the sha256 with:" -echo "echo "$TAR_FILE_HASH $BLOB_NAME" | sha256sum -c -" -rm "$TAR_FILE" \ No newline at end of file + az storage blob metadata update --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --metadata "sha256=$TAR_FILE_HASH" + + # Print the sha256sum. Downloaders will need to verify this + STORAGE_URL="$(az storage blob url --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --protocol "http")" + echo "You can now download the UTXO on $STORAGE_URL" + echo "Please, after download, verify the sha256 with:" + echo "echo "$TAR_FILE_HASH $BLOB_NAME" | sha256sum -c -" + rm "$TAR_FILE" +else + echo "SHA256: $TAR_FILE_HASH" + echo "File at: $TAR_FILE" +fi From 7eeb20f75c0961ae19388c403ea22e190bd0b1e5 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 00:40:11 +0900 Subject: [PATCH 07/36] Make the utxo-set scripts more friendly --- contrib/load-utxo-set.sh | 32 ++++++++++++++++++++-------- contrib/save-utxo-set-in-bitcoind.sh | 2 ++ contrib/save-utxo-set.sh | 11 +++++++--- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/contrib/load-utxo-set.sh b/contrib/load-utxo-set.sh index 8911311..23fd4d6 100755 --- a/contrib/load-utxo-set.sh +++ b/contrib/load-utxo-set.sh @@ -1,6 +1,11 @@ #!/bin/bash -# This script shows the steps to download and update an archive of the current UTXO Set +# This script shows the steps to download and deploy an archive of the current UTXO Set +# It will: +# 1. Download the UTXO Set from UTXO_DOWNLOAD_LINK, if UTXO_DOWNLOAD_LINK is empty, use NBITCOIN_NETWORK to find a default +# 2. Check the tarball against trusted hashes +# 3. Create the container's folders for blocks and chainstate, or empty them if they exists +# 4. Unzip the tarball if ! [ "$0" = "$BASH_SOURCE" ]; then echo "This script must not be sourced" @@ -12,11 +17,20 @@ if [[ $EUID -ne 0 ]]; then exit 1 fi -: "${UTXO_DOWNLOAD_LINK:=http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-mainnet-551636.tar}" -NETWORK="" -[[ $DOWNLOAD_LINK == *-testnet-* ]] && NETWORK="testnet" -[[ $DOWNLOAD_LINK == *-mainnet-* ]] && NETWORK="mainnet" -[[ $DOWNLOAD_LINK == *-regtest-* ]] && NETWORK="regtest" +if [[ "$NBITCOIN_NETWORK" ]]; then + echo "NBITCOIN_NETWORK should be set to mainnet, testnet or regtest" + exit 1 +fi + +if ! [[ "$UTXO_DOWNLOAD_LINK" ]]; then + [[ $NBITCOIN_NETWORK == "mainnet" ]] && UTXO_DOWNLOAD_LINK="http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-mainnet-551636.tar" + [[ $NBITCOIN_NETWORK == "testnet" ]] && UTXO_DOWNLOAD_LINK="http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-testnet-1445586.tar" +fi + +if ! [[ "$UTXO_DOWNLOAD_LINK" ]]; then + echo "No default UTXO_DOWNLOAD_LINK for $NBITCOIN_NETWORK" + exit 1 +fi BITCOIN_DATA_DIR="/var/lib/docker/volumes/generated_bitcoin_datadir/_data" [ ! -d "$BITCOIN_DATA_DIR" ] && mkdir -p "$BITCOIN_DATA_DIR" @@ -48,11 +62,11 @@ fi rm "sig.asc" cd - -NETWORK_DIRECTORY=$NETWORK -if [[ $NETWORK == "mainnet" ]]; then +NETWORK_DIRECTORY=$NBITCOIN_NETWORK +if [[ $NBITCOIN_NETWORK == "mainnet" ]]; then NETWORK_DIRECTORY="." fi -if [[ $NETWORK == "testnet" ]]; then +if [[ $NBITCOIN_NETWORK == "testnet" ]]; then NETWORK_DIRECTORY="testnet3" fi diff --git a/contrib/save-utxo-set-in-bitcoind.sh b/contrib/save-utxo-set-in-bitcoind.sh index 6ebfd73..72a8dec 100755 --- a/contrib/save-utxo-set-in-bitcoind.sh +++ b/contrib/save-utxo-set-in-bitcoind.sh @@ -1,3 +1,4 @@ +# This file is internal and meant to be run by save-utxo-set.sh BITCOIND="bitcoind -datadir=/data" BITCOIN_CLI="bitcoin-cli -datadir=/data" @@ -26,3 +27,4 @@ TAR_NAME="utxo-snapshot-bitcoin-$NBITCOIN_NETWORK-$PRUNED_HEIGHT.tar" echo "Creating $TAR_NAME..." tar -cf "$TAR_NAME" "$NETWORK_DIRECTORY/blocks/" tar -rf "$TAR_NAME" "$NETWORK_DIRECTORY/chainstate/" +exit diff --git a/contrib/save-utxo-set.sh b/contrib/save-utxo-set.sh index b713c94..35286ac 100755 --- a/contrib/save-utxo-set.sh +++ b/contrib/save-utxo-set.sh @@ -1,12 +1,17 @@ #!/bin/bash # This script shows the steps to create an archive of the current UTXO Set +# It will: +# 1. Shutdown BTCPay Server +# 2. Start bitcoind +# 3. Prune it to up to 289 blocks from the tip +# 4. Stop bitcoind +# 5. Archive in a tarball the blocks and chainstate directories +# 6. Restart BTCPay +# 7. If AZURE_STORAGE_CONNECTION_STRING is set, then upload to azure storage and make the blob public, else print hash and tarball : "${AZURE_STORAGE_CONTAINER:=public}" -# IN THE HOST ############################################################# - -# Stop btcpay btcpay-down.sh for i in /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*; do From 58ffc2b4667b9b94f7d6b8d0e6200ccc838e0ac7 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 00:41:26 +0900 Subject: [PATCH 08/36] Fix script --- contrib/load-utxo-set.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/load-utxo-set.sh b/contrib/load-utxo-set.sh index 23fd4d6..83c6983 100755 --- a/contrib/load-utxo-set.sh +++ b/contrib/load-utxo-set.sh @@ -17,7 +17,7 @@ if [[ $EUID -ne 0 ]]; then exit 1 fi -if [[ "$NBITCOIN_NETWORK" ]]; then +if ! [[ "$NBITCOIN_NETWORK" ]]; then echo "NBITCOIN_NETWORK should be set to mainnet, testnet or regtest" exit 1 fi From 54226a0b9b00087926f6a03ac5421e050bda53e0 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 00:45:22 +0900 Subject: [PATCH 09/36] Fix script --- contrib/load-utxo-set.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/load-utxo-set.sh b/contrib/load-utxo-set.sh index 83c6983..7c14ab4 100755 --- a/contrib/load-utxo-set.sh +++ b/contrib/load-utxo-set.sh @@ -35,13 +35,13 @@ fi BITCOIN_DATA_DIR="/var/lib/docker/volumes/generated_bitcoin_datadir/_data" [ ! -d "$BITCOIN_DATA_DIR" ] && mkdir -p "$BITCOIN_DATA_DIR" -TAR_NAME="$(basename $DOWNLOAD_LINK)" +TAR_NAME="$(basename $UTXO_DOWNLOAD_LINK)" TAR_FILE="$BITCOIN_DATA_DIR/$TAR_NAME" -echo "Downloading $DOWNLOAD_LINK to $TAR_FILE" cd "$BITCOIN_DATA_DIR" if [ ! -f "$TAR_FILE" ]; then - wget "$DOWNLOAD_LINK" -q --show-progress + echo "Downloading $UTXO_DOWNLOAD_LINK to $TAR_FILE" + wget "$UTXO_DOWNLOAD_LINK" -q --show-progress else echo "$TAR_FILE already exists" fi From ac1cff78fdf33d917ea87218b5ed42c5188fdc02 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 11:45:43 +0900 Subject: [PATCH 10/36] bump --- docker-compose-generator/docker-fragments/btcpayserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-generator/docker-fragments/btcpayserver.yml b/docker-compose-generator/docker-fragments/btcpayserver.yml index 293effb..0a5a0a0 100644 --- a/docker-compose-generator/docker-fragments/btcpayserver.yml +++ b/docker-compose-generator/docker-fragments/btcpayserver.yml @@ -4,7 +4,7 @@ services: btcpayserver: restart: unless-stopped - image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.17} + image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.19} expose: - "49392" environment: From 63f967a9497d7911686043eebb406cd1a4c2a747 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 12:50:55 +0900 Subject: [PATCH 11/36] Sign the UTXO sets hash --- contrib/load-utxo-set.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/contrib/load-utxo-set.sh b/contrib/load-utxo-set.sh index 7c14ab4..1b5c8ad 100755 --- a/contrib/load-utxo-set.sh +++ b/contrib/load-utxo-set.sh @@ -38,6 +38,7 @@ BITCOIN_DATA_DIR="/var/lib/docker/volumes/generated_bitcoin_datadir/_data" TAR_NAME="$(basename $UTXO_DOWNLOAD_LINK)" TAR_FILE="$BITCOIN_DATA_DIR/$TAR_NAME" +cp "utxo-sets" "$BITCOIN_DATA_DIR/utxo-sets" cd "$BITCOIN_DATA_DIR" if [ ! -f "$TAR_FILE" ]; then echo "Downloading $UTXO_DOWNLOAD_LINK to $TAR_FILE" @@ -46,20 +47,15 @@ else echo "$TAR_FILE already exists" fi -echo " -fab994299273080bf7124c8c45c4ada867974ca747900178496a69e450cf713f utxo-snapshot-bitcoin-mainnet-551636.tar -eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742 utxo-snapshot-bitcoin-testnet-1445586.tar -" > "trusted-utxo-sets.asc" - -grep "$TAR_NAME" "trusted-utxo-sets.asc" | tee "sig.asc" -rm "trusted-utxo-sets.asc" -if ! sha256sum -c "sig.asc"; then +grep "$TAR_NAME" "utxo-sets" | tee "utxo-set" +rm "utxo-sets" +if ! sha256sum -c "utxo-set"; then echo "$TAR_FILE is not trusted" - rm "sig.asc" + rm "utxo-set" cd - exit 1 fi -rm "sig.asc" +rm "utxo-set" cd - NETWORK_DIRECTORY=$NBITCOIN_NETWORK From c6f1f46800958efa1a5b51f402d6a3b388882f63 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 12:52:34 +0900 Subject: [PATCH 12/36] add signatures --- contrib/NicolasDorier.utxo-sets.asc | 21 +++++++++++++++++++++ contrib/utxo-sets | 2 ++ 2 files changed, 23 insertions(+) create mode 100644 contrib/NicolasDorier.utxo-sets.asc create mode 100644 contrib/utxo-sets diff --git a/contrib/NicolasDorier.utxo-sets.asc b/contrib/NicolasDorier.utxo-sets.asc new file mode 100644 index 0000000..e67907f --- /dev/null +++ b/contrib/NicolasDorier.utxo-sets.asc @@ -0,0 +1,21 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA256 + +fab994299273080bf7124c8c45c4ada867974ca747900178496a69e450cf713f utxo-snapshot-bitcoin-mainnet-551636.tar +eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742 utxo-snapshot-bitcoin-testnet-1445586.tar +-----BEGIN PGP SIGNATURE----- + +wsFcBAEBCAAQBQJcALGMCRBmGHY+8JGG/gAA0YsQAJgfTLHy/z+5c7pumb7QGl5y +afs7KrDUpc5jM6c6jeaKaysWm/aQmWXYtTYMFeN2+vexVLvbMiYSVMl5ts2zRScl +tY+HQ2aYlUP754S98gjFrWNner5wpREhu+UILtzB79ph/Baw2iOm0NJIsr3SA8B6 +VJ/JRRiaG4PW6TNJumiYVRZ7RCx+DnWveq+ombPDrjp7sbABx2s8tyIxFf7IW2wb +bOHA2FNjRu2+gd6PNVlNYVX+i0tx+q8f7yuCOxxSux5DvfK5p+TE2XqWoc3TbtqZ +9SXKeoUVDvhWjY7nH8D0LehYobh0LzCIpdsm1wqDpR3DOMlNAY+5OHA0+4BAoY+K +hPOyWA5TWLrlC6yMuz6ttVFvjOHGd7HwLqlU3NU4aj9oxLIjaHHHOcxvf+v0GrvL +QTd7YfyFeEpMQjwK/ueYlswxRqMHMFVLg/F0sAcwJ4W3F9yakz8mS341IDSWPoUR +Kc1KArFH4SqW2oc3ChzFwU/Om7rv5rZelJRRTzW8lJZFPysiLtUoEe1PeiYiikCL +Am40XX62EF+MG+TKQkYkTGlSPjNEE+ko9cQcAzMWVz1BZCUjcJH+dI/7uNsMOtp3 +y1ytX+Asv2KiLGAKzQvKBIrJzYuwuStVxUIZguJpb20bo8CkLOUKORjwsiQ4ktna +gHoN2GLHvDzOq4cixubg +=65Dw +-----END PGP SIGNATURE----- diff --git a/contrib/utxo-sets b/contrib/utxo-sets new file mode 100644 index 0000000..fbce729 --- /dev/null +++ b/contrib/utxo-sets @@ -0,0 +1,2 @@ +fab994299273080bf7124c8c45c4ada867974ca747900178496a69e450cf713f utxo-snapshot-bitcoin-mainnet-551636.tar +eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742 utxo-snapshot-bitcoin-testnet-1445586.tar \ No newline at end of file From b5e47f94cc138f5a8e1648ffb784bbf989f0aa5d Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 12:54:16 +0900 Subject: [PATCH 13/36] move files to FastSync --- contrib/{ => FastSync}/NicolasDorier.utxo-sets.asc | 0 contrib/{ => FastSync}/load-utxo-set.sh | 0 contrib/{ => FastSync}/save-utxo-set-in-bitcoind.sh | 0 contrib/{ => FastSync}/save-utxo-set.sh | 0 contrib/{ => FastSync}/utxo-sets | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename contrib/{ => FastSync}/NicolasDorier.utxo-sets.asc (100%) rename contrib/{ => FastSync}/load-utxo-set.sh (100%) rename contrib/{ => FastSync}/save-utxo-set-in-bitcoind.sh (100%) rename contrib/{ => FastSync}/save-utxo-set.sh (100%) rename contrib/{ => FastSync}/utxo-sets (100%) diff --git a/contrib/NicolasDorier.utxo-sets.asc b/contrib/FastSync/NicolasDorier.utxo-sets.asc similarity index 100% rename from contrib/NicolasDorier.utxo-sets.asc rename to contrib/FastSync/NicolasDorier.utxo-sets.asc diff --git a/contrib/load-utxo-set.sh b/contrib/FastSync/load-utxo-set.sh similarity index 100% rename from contrib/load-utxo-set.sh rename to contrib/FastSync/load-utxo-set.sh diff --git a/contrib/save-utxo-set-in-bitcoind.sh b/contrib/FastSync/save-utxo-set-in-bitcoind.sh similarity index 100% rename from contrib/save-utxo-set-in-bitcoind.sh rename to contrib/FastSync/save-utxo-set-in-bitcoind.sh diff --git a/contrib/save-utxo-set.sh b/contrib/FastSync/save-utxo-set.sh similarity index 100% rename from contrib/save-utxo-set.sh rename to contrib/FastSync/save-utxo-set.sh diff --git a/contrib/utxo-sets b/contrib/FastSync/utxo-sets similarity index 100% rename from contrib/utxo-sets rename to contrib/FastSync/utxo-sets From 8348d26166a48407f7a9be6abe51b417fbc9f05d Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 15:16:50 +0900 Subject: [PATCH 14/36] Document Fast Sync --- contrib/FastSync/README.md | 171 +++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 contrib/FastSync/README.md diff --git a/contrib/FastSync/README.md b/contrib/FastSync/README.md new file mode 100644 index 0000000..5049d8c --- /dev/null +++ b/contrib/FastSync/README.md @@ -0,0 +1,171 @@ +# Fast sync for Bitcoin + +## What problem does Fast Sync solve? + +When you start a new BTCPay Server, you need to synchronize your Bitcoin node from the genesis block. + +Validating from the genesis blocks takes in general 1 or 2 days on affordable servers. (around 10$ per months) + +However, on some low powered devices like raspberry PI, synchronization time will take around 2 weeks nowadays. (and it will linearly increase with time) + +Fast sync provides a solution to decrease dramatically the synchronization time to minutes or few hours. + +## How does Fast Sync solve it? + +In order for Bitcoin software to operate, you do not need all the history of blocks from the genesis. + +What you need is the state of Bitcoin up to a certain block (this state is called `UTXO Set`), and around 300 blocks before this point. + +Fast Sync downloads the UTXO Set at a specific block on an external server, and deploy it on your node. We call this file a `UTXO Set snapshot`. + +When your node start up again, it will only need to synchronize from the snapshot, to the latest blocks. + +## What are the downsides of Fast Sync? + +### Malicious UTXO Set + +Fast Sync can be potentially abused: +1. The attacker gives you an `UTXO Set snapshot` which does not follow Bitcoin consensus +2. The attacker makes a purchase to your store. +3. Nodes following the consensus would not recognize the payment as valid, but your node will. +4. The coins the attacker sent you are just worthless + +Other attacks can completely bring down your node. + +### Lightning Network routing issues + +As a merchant, you generally do not care about this issue. Merchants are mainly receiving payment, so they don't need to do any routing. + +However, if you plan to send payments from your lightning node, you may have issues: + +Because you do not have old blocks, then your lightning node won't see any channels which have been created prior to the snapshot. + +If this is a problem for you, just use an older snapshot. + +## How to verify that the UTXO Set snapshot follows the consensus? + +### If you trust the owner of this repository.... + +The snapshots recognized as valid by the `btcpayserver-docker` repository you cloned can be found on [utxo-sets](utxo-sets). + +The script [load-utxo-set.sh](load-utxo-set.sh) will download the utxo-set from the environment variable `UTXO_DOWNLOAD_LINK`. + +If `UTXO_DOWNLOAD_LINK` is empty, `NBITCOIN_NETWORK` (with value set to `mainnet` or `testnet`) will be used to take a default `UTXO_DOWNLOAD_LINK` that we hard coded inside [load-utxo-set.sh](load-utxo-set.sh). + +Once the files are downloaded, the hash will be checked against those in [utxo-sets](utxo-sets). + + +However: This only prove that `if the owner of this git repository is honest`, then the utxo-set are correct. + +NOTE: **Completing those steps does not mean that the UTXO set snapshot is legit**. It only mean that you trust the owner of this git repositoy to have verified that it is legit. + +### If you trust someone else... + +This repository contains the signatures of some developers, for example `NicolasDorier.utxo-sets.asc` contains the hashes that `NicolasDorier` verified himself. + +You need to verify with [KeyBase command line](https://keybase.io/docs/command_line) that the signature is legit: +```bash +keybase pgp verify -i NicolasDorier.utxo-sets.asc +``` +If you don't like command line, you can verify against [keybase verify page](https://keybase.io/verify) by just copying and pasting the content of [NicolasDorier.utxo-sets.asc](NicolasDorier.utxo-sets.asc). + + +You can verify that the handle `NicolasDorier` refers to the person who controls `NicolasDorier` twitter, github and reddit handle on [the keybase profile page](https://keybase.io/NicolasDorier). + +NOTE: **Completing those steps does not mean that the UTXO set snapshot is legit**. It only mean that you trust the owner of a Keybase account who has proved access to some social media accounts in the past. + +### Don't trust, verify! + +If you don't trust anybody, which should be the case as much as possible, then here are the steps to verify that the UTXO set is not malicious. + +1. You need another node that you own, `under your control`, that `you synchronized from the genesis block`. Let's call this node `Trusty`. +2. You need to create a new node which use `Fast Sync` with the UTXO snapshot you want to verify. Let's call this node, `Synchy`. +3. Wait that `Synchy` is fully synched. +4. Now on `Synchy` and `Trusty` run at the same time: + +```bash + bitcoin-cli gettxoutsetinfo +``` +If `Synchy` or `Trusty` are using BTCPay Server use: +```bash +bitcoin-cli.sh gettxoutsetinfo +``` + + +5. Verify that the output of `Synchy` and `Trusty` are **exactly** identical. + +If you have a trusted BTCPay Server node somewhere, you can ust `bitcoin-cli.sh gettxoutsetinfo` instead. + +NOTE: Completing those steps, under the assumption the software you are running is not malicious, **Prove that the UTXO set snapshot is legit**. + +## FAQ +### Can I add my signature to this repository? + +If you are a bitcoin developer or public figure, feel free to add your signature. For this you need: + +1. A [keybase account](http://keybase.io) linked to your social media accounts. +2. Follow the steps described in the [Don't trust, verify!](#donttrust) section each snapshots you want to sign. +3. Create a file with same format as [utxo-sets](utxo-sets) with the snapshots you validated. (Let's call this file `YOU.utxo-sets`) +4. Run the following command line + +``` +keybase pgp sign -i YOU.utxo-sets -c -t -o YOU.utxo-set.asc +rm YOU.utxo-set +git add YOU.utxo-set.asc +git commit -m "Add YOU utxo-set signature" -all +``` +And make a pull request to `btcpayserver-docker` repository. + +### Where can I download UTXO set snapshots + +You should not need to do this, because [load-utxo-set.sh](load-utxo-set.sh) do the hard work for you. + +But if you really want, just browse on [this listing](http://utxosets.blob.core.windows.net/public?restype=container&comp=list&include=metadata). + +Select the snapshot you want, and download it by querying `http://utxosets.blob.core.windows.net/public/{blobName}`. + +### How can I create my own snapshot? + +Assuming you have a node running on a docker deployment of BTCPay Server, you just need to run [save-utxo-set.sh](save-utxo-set.sh). + + This script shows the steps to create an archive of the current UTXO Set + It will: +1. Shutdown BTCPay Server +2. Start bitcoind +3. Prune it to up to 289 blocks from the tip +4. Stop bitcoind +5. Archive in a tarball the blocks and chainstate directories +6. Restart BTCPay +7. If `AZURE_STORAGE_CONNECTION_STRING` is set, then upload to azure storage and make the blob public, else print hash and tarball + +### How can I do this for my altcoin? + +Your altcoin does not need it, almost nobody use it compared to bitcoin. + +However, if you insist, follow what we did for Bitcoin, we can't hand hold you on this. + +### Do you plan to destroy Bitcoin? + +This feature may be controversial, because of the risk that almost nobody will follow the [Don't trust, verify!](#donttrust) step. + +What if somebody start spreading a corrupted snapshot on wild scale? + +I think this issue can be mitigated at the social layer. If several person start using social media for spreading their `bitcoin-cli getutxosetinfo` every 10 000 blocks, any corrupt snapshot would be soon detected. We plan to make expose the hash via `BTCPayServer` and make it easy for people to share. + +### Why you don't just: Run BTCPayServer as SPV + +All SPV solution brings a systemic risk to Bitcoin. If everybody relies on SPV to accept payment and miners want to change consensus rules, then you will have no leverage as individual, nor as a community to decide against. + +Even with `UTXO Set snapshots` you continue to validate consensus rules from the block of the snapshot. + +### Why you don't just: Run BTCPayServer an external node + +Why not just hosting BTCPayServer on the raspberry pi, but the bitcoin full node on another machine? + +For two reasons: + +First, `BTCPayServer` is trying to bring down the technical barriers to operate payments on your own. Running on an external node means that the user need the technical skills to set it up. + +`BTCPayServer` also relies on Bitcoin's RPC which is not meant to be exposed on internet. We can't see any simple enough solution which would allow normal people to run an external node somewhere else. + +The second reason is about reliability: You want your service to be self contained. If you host a node on another server, and for some reason this server goes down, then your `BTCPayServer` hosted on the raspberry PI will also cease to function. From 034e264d2e133acf40fc450ac2954c2b506e1de2 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 15:18:06 +0900 Subject: [PATCH 15/36] fix typo --- contrib/FastSync/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/FastSync/README.md b/contrib/FastSync/README.md index 5049d8c..497af05 100644 --- a/contrib/FastSync/README.md +++ b/contrib/FastSync/README.md @@ -152,13 +152,13 @@ What if somebody start spreading a corrupted snapshot on wild scale? I think this issue can be mitigated at the social layer. If several person start using social media for spreading their `bitcoin-cli getutxosetinfo` every 10 000 blocks, any corrupt snapshot would be soon detected. We plan to make expose the hash via `BTCPayServer` and make it easy for people to share. -### Why you don't just: Run BTCPayServer as SPV +### Why you don't just: Make BTCPayServer relies on SPV All SPV solution brings a systemic risk to Bitcoin. If everybody relies on SPV to accept payment and miners want to change consensus rules, then you will have no leverage as individual, nor as a community to decide against. Even with `UTXO Set snapshots` you continue to validate consensus rules from the block of the snapshot. -### Why you don't just: Run BTCPayServer an external node +### Why you don't just: Make BTCPayServer relies on an external trusted node Why not just hosting BTCPayServer on the raspberry pi, but the bitcoin full node on another machine? From 0d6c878be4d121b2c792d8dd474b7fe31eeb3e4e Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 15:18:21 +0900 Subject: [PATCH 16/36] fix typo --- contrib/FastSync/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/FastSync/README.md b/contrib/FastSync/README.md index 497af05..c6f738b 100644 --- a/contrib/FastSync/README.md +++ b/contrib/FastSync/README.md @@ -152,13 +152,13 @@ What if somebody start spreading a corrupted snapshot on wild scale? I think this issue can be mitigated at the social layer. If several person start using social media for spreading their `bitcoin-cli getutxosetinfo` every 10 000 blocks, any corrupt snapshot would be soon detected. We plan to make expose the hash via `BTCPayServer` and make it easy for people to share. -### Why you don't just: Make BTCPayServer relies on SPV +### Why you don't just: Make BTCPayServer rely on SPV All SPV solution brings a systemic risk to Bitcoin. If everybody relies on SPV to accept payment and miners want to change consensus rules, then you will have no leverage as individual, nor as a community to decide against. Even with `UTXO Set snapshots` you continue to validate consensus rules from the block of the snapshot. -### Why you don't just: Make BTCPayServer relies on an external trusted node +### Why you don't just: Make BTCPayServer rely on an external trusted node Why not just hosting BTCPayServer on the raspberry pi, but the bitcoin full node on another machine? From 87bdc3dfdc6fbacf8effbd23775304d7a698773f Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 15:25:11 +0900 Subject: [PATCH 17/36] Remove repeated --- contrib/FastSync/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/FastSync/README.md b/contrib/FastSync/README.md index c6f738b..2fc4005 100644 --- a/contrib/FastSync/README.md +++ b/contrib/FastSync/README.md @@ -94,8 +94,6 @@ bitcoin-cli.sh gettxoutsetinfo 5. Verify that the output of `Synchy` and `Trusty` are **exactly** identical. -If you have a trusted BTCPay Server node somewhere, you can ust `bitcoin-cli.sh gettxoutsetinfo` instead. - NOTE: Completing those steps, under the assumption the software you are running is not malicious, **Prove that the UTXO set snapshot is legit**. ## FAQ From 2023516a3826ac6417b7fc56d4ebc6850f573837 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 15:26:01 +0900 Subject: [PATCH 18/36] Fix typo --- contrib/FastSync/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/FastSync/README.md b/contrib/FastSync/README.md index 2fc4005..a5777b0 100644 --- a/contrib/FastSync/README.md +++ b/contrib/FastSync/README.md @@ -94,7 +94,7 @@ bitcoin-cli.sh gettxoutsetinfo 5. Verify that the output of `Synchy` and `Trusty` are **exactly** identical. -NOTE: Completing those steps, under the assumption the software you are running is not malicious, **Prove that the UTXO set snapshot is legit**. +NOTE: Completing those steps, under the assumption the software you are running is not malicious, **correctly prove that the UTXO set snapshot is legit**. ## FAQ ### Can I add my signature to this repository? From 0ecf274decd4485c1cb3f45921c68231d233c554 Mon Sep 17 00:00:00 2001 From: rockstardev Date: Fri, 30 Nov 2018 00:59:16 -0600 Subject: [PATCH 19/36] Signing UTXO sets, rockstardev --- contrib/FastSync/rockstardev.utxo-set.asc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 contrib/FastSync/rockstardev.utxo-set.asc diff --git a/contrib/FastSync/rockstardev.utxo-set.asc b/contrib/FastSync/rockstardev.utxo-set.asc new file mode 100644 index 0000000..d694666 --- /dev/null +++ b/contrib/FastSync/rockstardev.utxo-set.asc @@ -0,0 +1,21 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA256 + +fab994299273080bf7124c8c45c4ada867974ca747900178496a69e450cf713f utxo-snapshot-bitcoin-mainnet-551636.tar +eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742 utxo-snapshot-bitcoin-testnet-1445586.tar +-----BEGIN PGP SIGNATURE----- + +wsFcBAEBCAAQBQJcAN7+CRB5wjOsK6943QAAvx8QADXzOsU/tfn0h2fDwNWCVI+D +VxoWCc+WshOu7vAH0qm/oUSjs99+FS0nP6eHSsBy+vO6h6cdI6CC8I062faAy/oH +8eyH1WIxjLs8cXySImR60M804u7906Jmf9ouUu0jqlNJVHE34utIXfrzhlf3cZoj +737Y9gRDLYZuiFK7h9H1hInD8h9CpkOachglbBKjErZ/E53q3EVxk6jjbOp1fvfk +2KekrZThiUHa0aRD1OfyDiCA611TsAplfF/2OpXeBB8yQKm2qgkz+xLXW68noQnC +0ErxAHycqgvgeYKGZr9JQNCBvrv9ArZY15TJiGkse2P/YmisEOpePzh16kQkJARU +W5ew3cCOwDnhUtV1oZAoVLu75XlIdRn404EKjHr7JbwlT9RI1EsFb+ESG9wtiTur +a743yczsNXePfD1yO/nm2n6O/CW4HeKMDbKakL0FT3hLaoo3Yc72YItJw93zVWdL +tO7aCRFtBXKUGG1rAbvXQIBr4JDEMq4H8/aE8dBJXneRn9wMlFVEpM5Yi9OMzf1M +QoelL0vkj79Hfx13jQCAY/z3gadZ12XNMlsWembPFUpSxGwGEDra41olsUkjedcm +0ZAHzRjIXcY18HSxEHjqiOnU2lFaBZffXr8GlUgMmUepEV/MbKkgL9JOo+6JsizF +tYLN91o5MaDkR4Tf2TCj +=aSFC +-----END PGP SIGNATURE----- From 37a63303d2ba58748b1c9f7baddd395e0944f7e1 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 16:31:15 +0900 Subject: [PATCH 20/36] fix typo --- contrib/FastSync/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/FastSync/README.md b/contrib/FastSync/README.md index a5777b0..56a1109 100644 --- a/contrib/FastSync/README.md +++ b/contrib/FastSync/README.md @@ -107,9 +107,9 @@ If you are a bitcoin developer or public figure, feel free to add your signature 4. Run the following command line ``` -keybase pgp sign -i YOU.utxo-sets -c -t -o YOU.utxo-set.asc -rm YOU.utxo-set -git add YOU.utxo-set.asc +keybase pgp sign -i YOU.utxo-sets -c -t -o YOU.utxo-sets.asc +rm YOU.utxo-sets +git add YOU.utxo-sets.asc git commit -m "Add YOU utxo-set signature" -all ``` And make a pull request to `btcpayserver-docker` repository. From 2718d301e41f935306a0c0d10ee9d682039352b8 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 16:32:16 +0900 Subject: [PATCH 21/36] rename rockstardev sets --- .../{rockstardev.utxo-set.asc => rockstardev.utxo-sets.asc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename contrib/FastSync/{rockstardev.utxo-set.asc => rockstardev.utxo-sets.asc} (100%) diff --git a/contrib/FastSync/rockstardev.utxo-set.asc b/contrib/FastSync/rockstardev.utxo-sets.asc similarity index 100% rename from contrib/FastSync/rockstardev.utxo-set.asc rename to contrib/FastSync/rockstardev.utxo-sets.asc From 0d9a68a88884898fadacf008bb0182f83a50c255 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 16:42:01 +0900 Subject: [PATCH 22/36] move sigs in different directory --- contrib/FastSync/README.md | 13 +++++++------ .../FastSync/{ => sigs}/NicolasDorier.utxo-sets.asc | 0 .../FastSync/{ => sigs}/rockstardev.utxo-sets.asc | 0 3 files changed, 7 insertions(+), 6 deletions(-) rename contrib/FastSync/{ => sigs}/NicolasDorier.utxo-sets.asc (100%) rename contrib/FastSync/{ => sigs}/rockstardev.utxo-sets.asc (100%) diff --git a/contrib/FastSync/README.md b/contrib/FastSync/README.md index 56a1109..faca61e 100644 --- a/contrib/FastSync/README.md +++ b/contrib/FastSync/README.md @@ -61,13 +61,13 @@ NOTE: **Completing those steps does not mean that the UTXO set snapshot is legit ### If you trust someone else... -This repository contains the signatures of some developers, for example `NicolasDorier.utxo-sets.asc` contains the hashes that `NicolasDorier` verified himself. +This repository contains the signatures of some developers, for example [sigs/NicolasDorier.utxo-sets.asc](sigs/NicolasDorier.utxo-sets.asc) contains the hashes that `NicolasDorier` verified himself. You need to verify with [KeyBase command line](https://keybase.io/docs/command_line) that the signature is legit: ```bash -keybase pgp verify -i NicolasDorier.utxo-sets.asc +keybase pgp verify -i sigs/NicolasDorier.utxo-sets.asc ``` -If you don't like command line, you can verify against [keybase verify page](https://keybase.io/verify) by just copying and pasting the content of [NicolasDorier.utxo-sets.asc](NicolasDorier.utxo-sets.asc). +If you don't like command line, you can verify against [keybase verify page](https://keybase.io/verify) by just copying and pasting the content of [sigs/NicolasDorier.utxo-sets.asc](sigs/NicolasDorier.utxo-sets.asc). You can verify that the handle `NicolasDorier` refers to the person who controls `NicolasDorier` twitter, github and reddit handle on [the keybase profile page](https://keybase.io/NicolasDorier). @@ -106,10 +106,11 @@ If you are a bitcoin developer or public figure, feel free to add your signature 3. Create a file with same format as [utxo-sets](utxo-sets) with the snapshots you validated. (Let's call this file `YOU.utxo-sets`) 4. Run the following command line -``` -keybase pgp sign -i YOU.utxo-sets -c -t -o YOU.utxo-sets.asc +```bash +# Assuming your are inside the FastSync directory +keybase pgp sign -i YOU.utxo-sets -c -t -o sigs/YOU.utxo-sets.asc rm YOU.utxo-sets -git add YOU.utxo-sets.asc +git add sigs/YOU.utxo-sets.asc git commit -m "Add YOU utxo-set signature" -all ``` And make a pull request to `btcpayserver-docker` repository. diff --git a/contrib/FastSync/NicolasDorier.utxo-sets.asc b/contrib/FastSync/sigs/NicolasDorier.utxo-sets.asc similarity index 100% rename from contrib/FastSync/NicolasDorier.utxo-sets.asc rename to contrib/FastSync/sigs/NicolasDorier.utxo-sets.asc diff --git a/contrib/FastSync/rockstardev.utxo-sets.asc b/contrib/FastSync/sigs/rockstardev.utxo-sets.asc similarity index 100% rename from contrib/FastSync/rockstardev.utxo-sets.asc rename to contrib/FastSync/sigs/rockstardev.utxo-sets.asc From 080fbdaa9c6cd3c4b4f57415ccbbc9e3fc9b5749 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 30 Nov 2018 17:19:41 +0900 Subject: [PATCH 23/36] bump docker-compose --- btcpay-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/btcpay-setup.sh b/btcpay-setup.sh index 026c846..60bfe15 100755 --- a/btcpay-setup.sh +++ b/btcpay-setup.sh @@ -234,14 +234,14 @@ if ! [ -x "$(command -v docker)" ] || ! [ -x "$(command -v docker-compose)" ]; t fi if ! [ -x "$(command -v docker-compose)" ]; then if [[ "$(uname -m)" == "x86_64" ]]; then - DOCKER_COMPOSE_DOWNLOAD="https://github.com/docker/compose/releases/download/1.17.1/docker-compose-$(uname -s)-$(uname -m)" + DOCKER_COMPOSE_DOWNLOAD="https://github.com/docker/compose/releases/download/1.23.2/docker-compose-`uname -s`-`uname -m`" echo "Trying to install docker-compose by downloading on $DOCKER_COMPOSE_DOWNLOAD ($(uname -m))" curl -L "$DOCKER_COMPOSE_DOWNLOAD" -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose else echo "Trying to install docker-compose by using the docker-compose-builder ($(uname -m))" ! [ -d "dist" ] && mkdir dist - docker run --rm -ti -v "$(pwd)/dist:/dist" btcpayserver/docker-compose-builder:1.22.0 + docker run --rm -ti -v "$(pwd)/dist:/dist" btcpayserver/docker-compose-builder:1.23.2 mv dist/docker-compose /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose rm -rf "dist" From 8c9d9f091006e7cb638c5f3323cd6299cde11c02 Mon Sep 17 00:00:00 2001 From: wakiyamap Date: Fri, 30 Nov 2018 05:00:14 -0500 Subject: [PATCH 24/36] support monacoin --- btcpay-setup.sh | 2 +- .../docker-fragments/monacoin.yml | 34 +++++++++++++++++++ .../docker-fragments/opt-save-memory.yml | 7 +++- .../docker-fragments/opt-save-storage-s.yml | 3 ++ .../docker-fragments/opt-save-storage-xs.yml | 3 ++ .../docker-fragments/opt-save-storage-xxs.yml | 3 ++ .../docker-fragments/opt-save-storage.yml | 5 ++- .../src/CryptoDefinition.cs | 7 +++- monacoin-cli.ps1 | 1 + monacoin-cli.sh | 3 ++ 10 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 docker-compose-generator/docker-fragments/monacoin.yml create mode 100755 monacoin-cli.ps1 create mode 100755 monacoin-cli.sh diff --git a/btcpay-setup.sh b/btcpay-setup.sh index 60bfe15..ccd480e 100755 --- a/btcpay-setup.sh +++ b/btcpay-setup.sh @@ -52,7 +52,7 @@ Environment variables: LETSENCRYPT_EMAIL: A mail will be sent to this address if certificate expires and fail to renew automatically (eg. me@example.com) NBITCOIN_NETWORK: The type of network to use (eg. mainnet, testnet or regtest. Default: mainnet) LIGHTNING_ALIAS: An alias for your lightning network node if used - BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btg, grs, ftc, via, none. Default: btc) + BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btg, grs, ftc, via, mona, none. Default: btc) BTCPAYGEN_CRYPTO2: Second supported crypto currency (Default: empty) BTCPAYGEN_CRYPTON: N th supported crypto currency where N is maximum at maximum 9. (Default: none) BTCPAYGEN_REVERSEPROXY: Whether to use or not a reverse proxy. NGinx setup HTTPS for you. (eg. nginx, traefik, none. Default: nginx) diff --git a/docker-compose-generator/docker-fragments/monacoin.yml b/docker-compose-generator/docker-fragments/monacoin.yml new file mode 100644 index 0000000..6ca9d08 --- /dev/null +++ b/docker-compose-generator/docker-fragments/monacoin.yml @@ -0,0 +1,34 @@ +version: "3" + +services: + monacoind: + restart: unless-stopped + container_name: btcpayserver_monacoind + image: nicolasdorier/docker-monacoin:0.16.3 + environment: + BITCOIN_EXTRA_ARGS: | + rpcport=43782 + ${NBITCOIN_NETWORK:-regtest}=1 + port=39388 + whitelist=0.0.0.0/0 + expose: + - "43782" + - "39388" + volumes: + - "monacoin_datadir:/data" + nbxplorer: + environment: + NBXPLORER_CHAINS: "mona" + NBXPLORER_MONARPCURL: http://monacoind:43782/ + NBXPLORER_MONANODEENDPOINT: monacoind:39388 + links: + - monacoind + volumes: + - "monacoin_datadir:/root/.monacoin" + btcpayserver: + environment: + BTCPAY_MONAEXPLORERURL: http://nbxplorer:32838/ + BTCPAY_CHAINS: "mona" + +volumes: + monacoin_datadir: diff --git a/docker-compose-generator/docker-fragments/opt-save-memory.yml b/docker-compose-generator/docker-fragments/opt-save-memory.yml index 90a4479..797c58f 100644 --- a/docker-compose-generator/docker-fragments/opt-save-memory.yml +++ b/docker-compose-generator/docker-fragments/opt-save-memory.yml @@ -31,4 +31,9 @@ services: environment: BITCOIN_EXTRA_ARGS: | dbcache=150 - maxmempool=100 \ No newline at end of file + maxmempool=100 + monacoind: + environment: + BITCOIN_EXTRA_ARGS: | + dbcache=150 + maxmempool=100 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-s.yml b/docker-compose-generator/docker-fragments/opt-save-storage-s.yml index 9acf2c0..d4dd57e 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-s.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-s.yml @@ -24,3 +24,6 @@ services: dogecoind: environment: DOGECOIN_EXTRA_ARGS: prune=50000 + monacoind: + environment: + BITCOIN_EXTRA_ARGS: prune=50000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml b/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml index 7b95f20..a413ccc 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml @@ -24,3 +24,6 @@ services: dogecoind: environment: DOGECOIN_EXTRA_ARGS: prune=25000 + monacoind: + environment: + BITCOIN_EXTRA_ARGS: prune=25000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml b/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml index 4e8ce5f..30456bd 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml @@ -24,3 +24,6 @@ services: dogecoind: environment: DOGECOIN_EXTRA_ARGS: prune=5000 + monacoind: + environment: + BITCOIN_EXTRA_ARGS: prune=5000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage.yml b/docker-compose-generator/docker-fragments/opt-save-storage.yml index 4a88674..72dbe25 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage.yml @@ -23,4 +23,7 @@ services: BITCOIN_EXTRA_ARGS: prune=100000 dogecoind: environment: - DOGECOIN_EXTRA_ARGS: prune=100000 \ No newline at end of file + DOGECOIN_EXTRA_ARGS: prune=100000 + monacoind: + environment: + BITCOIN_EXTRA_ARGS: prune=100000 diff --git a/docker-compose-generator/src/CryptoDefinition.cs b/docker-compose-generator/src/CryptoDefinition.cs index 4798b56..4825ec3 100644 --- a/docker-compose-generator/src/CryptoDefinition.cs +++ b/docker-compose-generator/src/CryptoDefinition.cs @@ -66,10 +66,15 @@ namespace DockerGenerator Crypto = "via", CryptoFragment = "viacoin" }, - new CryptoDefinition() + new CryptoDefinition() { Crypto = "doge", CryptoFragment = "dogecoin" + }, + new CryptoDefinition() + { + Crypto = "mona", + CryptoFragment = "monacoin" } }; } diff --git a/monacoin-cli.ps1 b/monacoin-cli.ps1 new file mode 100755 index 0000000..4985daf --- /dev/null +++ b/monacoin-cli.ps1 @@ -0,0 +1 @@ +docker exec -ti btcpayserver_monacoind monacoin-cli -datadir="/data" $args diff --git a/monacoin-cli.sh b/monacoin-cli.sh new file mode 100755 index 0000000..c289ead --- /dev/null +++ b/monacoin-cli.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -ti btcpayserver_monacoind monacoin-cli -datadir="/data" "$@" From c4f80e6f2ee06bb3181a061371f5ec681a802db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=84=87=E5=B1=B1P?= Date: Fri, 30 Nov 2018 20:14:38 +0900 Subject: [PATCH 25/36] fix typo --- docker-compose-generator/docker-fragments/monacoin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-generator/docker-fragments/monacoin.yml b/docker-compose-generator/docker-fragments/monacoin.yml index 6ca9d08..c9f788e 100644 --- a/docker-compose-generator/docker-fragments/monacoin.yml +++ b/docker-compose-generator/docker-fragments/monacoin.yml @@ -4,7 +4,7 @@ services: monacoind: restart: unless-stopped container_name: btcpayserver_monacoind - image: nicolasdorier/docker-monacoin:0.16.3 + image: wakiyamap/docker-monacoin:0.16.3 environment: BITCOIN_EXTRA_ARGS: | rpcport=43782 From 71ced0c11f5aaed0ff6ad34e082b2606e2b4161b Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Sat, 1 Dec 2018 11:49:33 +0900 Subject: [PATCH 26/36] Disable bitcoin wallet (interference with Fast Sync) --- docker-compose-generator/docker-fragments/bitcoin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-generator/docker-fragments/bitcoin.yml b/docker-compose-generator/docker-fragments/bitcoin.yml index 84a10ea..b56ccf5 100644 --- a/docker-compose-generator/docker-fragments/bitcoin.yml +++ b/docker-compose-generator/docker-fragments/bitcoin.yml @@ -8,9 +8,9 @@ services: environment: BITCOIN_NETWORK: ${NBITCOIN_NETWORK:-regtest} BITCOIN_EXTRA_ARGS: | - deprecatedrpc=signrawtransaction rpcport=43782 port=39388 + disablewallet=1 whitelist=0.0.0.0/0 expose: - "43782" From ad578deefb462c80354f489eeb7b1ef4710b1759 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Sat, 1 Dec 2018 13:26:26 +0900 Subject: [PATCH 27/36] bump --- docker-compose-generator/docker-fragments/btcpayserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-generator/docker-fragments/btcpayserver.yml b/docker-compose-generator/docker-fragments/btcpayserver.yml index 0a5a0a0..9a17bfc 100644 --- a/docker-compose-generator/docker-fragments/btcpayserver.yml +++ b/docker-compose-generator/docker-fragments/btcpayserver.yml @@ -4,7 +4,7 @@ services: btcpayserver: restart: unless-stopped - image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.19} + image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.21} expose: - "49392" environment: From a52b65e459cda9e88fdd207e01d3853ed4df56d6 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Sat, 1 Dec 2018 22:13:50 +0900 Subject: [PATCH 28/36] add doge to the list of supported coins --- btcpay-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btcpay-setup.sh b/btcpay-setup.sh index ccd480e..54c7e83 100755 --- a/btcpay-setup.sh +++ b/btcpay-setup.sh @@ -52,7 +52,7 @@ Environment variables: LETSENCRYPT_EMAIL: A mail will be sent to this address if certificate expires and fail to renew automatically (eg. me@example.com) NBITCOIN_NETWORK: The type of network to use (eg. mainnet, testnet or regtest. Default: mainnet) LIGHTNING_ALIAS: An alias for your lightning network node if used - BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btg, grs, ftc, via, mona, none. Default: btc) + BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btg, grs, ftc, via, doge, mona, none. Default: btc) BTCPAYGEN_CRYPTO2: Second supported crypto currency (Default: empty) BTCPAYGEN_CRYPTON: N th supported crypto currency where N is maximum at maximum 9. (Default: none) BTCPAYGEN_REVERSEPROXY: Whether to use or not a reverse proxy. NGinx setup HTTPS for you. (eg. nginx, traefik, none. Default: nginx) From a5137f7100d8adda953897d987955c1253f745d2 Mon Sep 17 00:00:00 2001 From: David B Date: Mon, 20 Aug 2018 23:08:10 +0000 Subject: [PATCH 29/36] Add BTX support --- bitcore-cli.ps1 | 1 + bitcore-cli.sh | 3 ++ btcpay-setup.sh | 2 +- .../docker-fragments/bitcore.yml | 34 +++++++++++++++++++ .../docker-fragments/opt-save-storage-s.yml | 3 ++ .../docker-fragments/opt-save-storage-xs.yml | 3 ++ .../docker-fragments/opt-save-storage-xxs.yml | 3 ++ .../docker-fragments/opt-save-storage.yml | 3 ++ .../src/CryptoDefinition.cs | 5 +++ 9 files changed, 56 insertions(+), 1 deletion(-) create mode 100755 bitcore-cli.ps1 create mode 100755 bitcore-cli.sh create mode 100644 docker-compose-generator/docker-fragments/bitcore.yml diff --git a/bitcore-cli.ps1 b/bitcore-cli.ps1 new file mode 100755 index 0000000..49bec2e --- /dev/null +++ b/bitcore-cli.ps1 @@ -0,0 +1 @@ +docker exec -ti btcpayserver_bitcored bitcore-cli -datadir="/data" $args diff --git a/bitcore-cli.sh b/bitcore-cli.sh new file mode 100755 index 0000000..1eb73fa --- /dev/null +++ b/bitcore-cli.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -ti btcpayserver_bitcored bitcore-cli -datadir="/data" "$@" diff --git a/btcpay-setup.sh b/btcpay-setup.sh index 54c7e83..3552783 100755 --- a/btcpay-setup.sh +++ b/btcpay-setup.sh @@ -52,7 +52,7 @@ Environment variables: LETSENCRYPT_EMAIL: A mail will be sent to this address if certificate expires and fail to renew automatically (eg. me@example.com) NBITCOIN_NETWORK: The type of network to use (eg. mainnet, testnet or regtest. Default: mainnet) LIGHTNING_ALIAS: An alias for your lightning network node if used - BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btg, grs, ftc, via, doge, mona, none. Default: btc) + BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btx, btg, grs, ftc, via, doge, mona, none. Default: btc) BTCPAYGEN_CRYPTO2: Second supported crypto currency (Default: empty) BTCPAYGEN_CRYPTON: N th supported crypto currency where N is maximum at maximum 9. (Default: none) BTCPAYGEN_REVERSEPROXY: Whether to use or not a reverse proxy. NGinx setup HTTPS for you. (eg. nginx, traefik, none. Default: nginx) diff --git a/docker-compose-generator/docker-fragments/bitcore.yml b/docker-compose-generator/docker-fragments/bitcore.yml new file mode 100644 index 0000000..db6d860 --- /dev/null +++ b/docker-compose-generator/docker-fragments/bitcore.yml @@ -0,0 +1,34 @@ +version: "3" + +services: + bitcored: + restart: unless-stopped + container_name: btcpayserver_bitcored + image: dalijolijo/docker-bitcore:0.15.2 + environment: + BITCOIN_EXTRA_ARGS: | + rpcport=43782 + ${NBITCOIN_NETWORK:-regtest}=1 + port=39388 + whitelist=0.0.0.0/0 + expose: + - "43782" + - "39388" + volumes: + - "bitcore_datadir:/data" + nbxplorer: + environment: + NBXPLORER_CHAINS: "btx" + NBXPLORER_BTXRPCURL: http://bitcored:43782/ + NBXPLORER_BTXNODEENDPOINT: bitcored:39388 + links: + - bitcored + volumes: + - "bitcore_datadir:/root/.bitcore" + btcpayserver: + environment: + BTCPAY_BTXEXPLORERURL: http://nbxplorer:32838/ + BTCPAY_CHAINS: "btx" + +volumes: + bitcore_datadir: diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-s.yml b/docker-compose-generator/docker-fragments/opt-save-storage-s.yml index d4dd57e..839d1eb 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-s.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-s.yml @@ -6,6 +6,9 @@ services: bitcoind: environment: BITCOIN_EXTRA_ARGS: prune=50000 + bitcored: + environment: + BITCOIN_EXTRA_ARGS: prune=50000 bgoldd: environment: BITCOIN_EXTRA_ARGS: prune=50000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml b/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml index a413ccc..7842263 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-xs.yml @@ -6,6 +6,9 @@ services: bitcoind: environment: BITCOIN_EXTRA_ARGS: prune=25000 + bitcored: + environment: + BITCOIN_EXTRA_ARGS: prune=25000 bgoldd: environment: BITCOIN_EXTRA_ARGS: prune=25000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml b/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml index 30456bd..240b54e 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml @@ -6,6 +6,9 @@ services: bitcoind: environment: BITCOIN_EXTRA_ARGS: prune=5000 + bitcored: + environment: + BITCOIN_EXTRA_ARGS: prune=5000 bgoldd: environment: BITCOIN_EXTRA_ARGS: prune=5000 diff --git a/docker-compose-generator/docker-fragments/opt-save-storage.yml b/docker-compose-generator/docker-fragments/opt-save-storage.yml index 72dbe25..52695d6 100644 --- a/docker-compose-generator/docker-fragments/opt-save-storage.yml +++ b/docker-compose-generator/docker-fragments/opt-save-storage.yml @@ -6,6 +6,9 @@ services: bitcoind: environment: BITCOIN_EXTRA_ARGS: prune=100000 + bitcored: + environment: + BITCOIN_EXTRA_ARGS: prune=100000 bgoldd: environment: BITCOIN_EXTRA_ARGS: prune=100000 diff --git a/docker-compose-generator/src/CryptoDefinition.cs b/docker-compose-generator/src/CryptoDefinition.cs index 4825ec3..5aa92fe 100644 --- a/docker-compose-generator/src/CryptoDefinition.cs +++ b/docker-compose-generator/src/CryptoDefinition.cs @@ -46,6 +46,11 @@ namespace DockerGenerator LNDFragment = "bitcoin-lnd" }, new CryptoDefinition() + { + Crypto = "btx", + CryptoFragment = "bitcore", + }, + new CryptoDefinition() { Crypto = "btg", CryptoFragment = "bgold", From 9e068bdb14c8f91f96e7582b3a7e7c895ee55aa4 Mon Sep 17 00:00:00 2001 From: David <37987466+dalijolijo@users.noreply.github.com> Date: Sat, 1 Dec 2018 15:55:16 +0100 Subject: [PATCH 30/36] bump --- docker-compose-generator/docker-fragments/nbxplorer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-generator/docker-fragments/nbxplorer.yml b/docker-compose-generator/docker-fragments/nbxplorer.yml index 511d1af..46e8738 100644 --- a/docker-compose-generator/docker-fragments/nbxplorer.yml +++ b/docker-compose-generator/docker-fragments/nbxplorer.yml @@ -4,7 +4,7 @@ services: nbxplorer: restart: unless-stopped - image: nicolasdorier/nbxplorer:2.0.0.1 + image: nicolasdorier/nbxplorer:2.0.0.2 expose: - "32838" environment: @@ -14,4 +14,4 @@ services: - "nbxplorer_datadir:/datadir" volumes: - nbxplorer_datadir: \ No newline at end of file + nbxplorer_datadir: From c9a0454034886778be8364515eb17abf8b041aec Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Sun, 2 Dec 2018 13:32:01 +0900 Subject: [PATCH 31/36] Add script and instruction to build all the docker images by yourself --- contrib/DockerFileBuildHelper/DockerFile.cs | 28 ++ .../DockerFileBuildHelper.csproj | 10 + contrib/DockerFileBuildHelper/DockerInfo.cs | 22 ++ contrib/DockerFileBuildHelper/Extensions.cs | 19 ++ contrib/DockerFileBuildHelper/Image.cs | 55 ++++ contrib/DockerFileBuildHelper/Program.cs | 271 ++++++++++++++++++ contrib/DockerFileBuildHelper/README.md | 16 ++ contrib/DockerFileBuildHelper/run.sh | 3 + contrib/DockerFileBuildHelper/update-repo.ps1 | 1 + contrib/DockerFileBuildHelper/update-repo.sh | 3 + contrib/build-all-images.sh | 202 +++++++++++++ 11 files changed, 630 insertions(+) create mode 100644 contrib/DockerFileBuildHelper/DockerFile.cs create mode 100644 contrib/DockerFileBuildHelper/DockerFileBuildHelper.csproj create mode 100644 contrib/DockerFileBuildHelper/DockerInfo.cs create mode 100644 contrib/DockerFileBuildHelper/Extensions.cs create mode 100644 contrib/DockerFileBuildHelper/Image.cs create mode 100644 contrib/DockerFileBuildHelper/Program.cs create mode 100644 contrib/DockerFileBuildHelper/README.md create mode 100755 contrib/DockerFileBuildHelper/run.sh create mode 100755 contrib/DockerFileBuildHelper/update-repo.ps1 create mode 100755 contrib/DockerFileBuildHelper/update-repo.sh create mode 100755 contrib/build-all-images.sh diff --git a/contrib/DockerFileBuildHelper/DockerFile.cs b/contrib/DockerFileBuildHelper/DockerFile.cs new file mode 100644 index 0000000..b1d7d82 --- /dev/null +++ b/contrib/DockerFileBuildHelper/DockerFile.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DockerFileBuildHelper +{ + public class DockerFile + { + public string DockerFileName { get; private set; } + public string DockerFilePath { get; private set; } + + public static DockerFile Parse(string str) + { + var file = new DockerFile(); + var lastPart = str.LastIndexOf('/'); + file.DockerFileName = str.Substring(lastPart + 1); + if (lastPart == -1) + { + file.DockerFilePath = "."; + } + else + { + file.DockerFilePath = str.Substring(0, lastPart); + } + return file; + } + } +} diff --git a/contrib/DockerFileBuildHelper/DockerFileBuildHelper.csproj b/contrib/DockerFileBuildHelper/DockerFileBuildHelper.csproj new file mode 100644 index 0000000..5a77ed4 --- /dev/null +++ b/contrib/DockerFileBuildHelper/DockerFileBuildHelper.csproj @@ -0,0 +1,10 @@ + + + + Exe + netcoreapp2.1 + + + + + diff --git a/contrib/DockerFileBuildHelper/DockerInfo.cs b/contrib/DockerFileBuildHelper/DockerInfo.cs new file mode 100644 index 0000000..ace3445 --- /dev/null +++ b/contrib/DockerFileBuildHelper/DockerInfo.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DockerFileBuildHelper +{ + public class DockerInfo + { + public string DockerFilePath { get; set; } + public string DockerFilePathARM32v7 { get; set; } + public string DockerFilePathARM64v8 { get; set; } + public string DockerHubLink { get; set; } + public string GitLink { get; set; } + public string GitRef { get; set; } + public Image Image { get; internal set; } + + public string GetGithubLinkOf(string path) + { + return $"https://raw.githubusercontent.com/{GitLink.Substring("https://github.com/".Length)}/{GitRef}/{path}"; + } + } +} diff --git a/contrib/DockerFileBuildHelper/Extensions.cs b/contrib/DockerFileBuildHelper/Extensions.cs new file mode 100644 index 0000000..f645a48 --- /dev/null +++ b/contrib/DockerFileBuildHelper/Extensions.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; +using YamlDotNet.RepresentationModel; + +namespace DockerFileBuildHelper +{ + public static class Extensions + { + public static YamlNode TryGet(this YamlNode node, string key) + { + try + { + return node[key]; + } + catch (KeyNotFoundException) { return null; } + } + } +} diff --git a/contrib/DockerFileBuildHelper/Image.cs b/contrib/DockerFileBuildHelper/Image.cs new file mode 100644 index 0000000..7733b79 --- /dev/null +++ b/contrib/DockerFileBuildHelper/Image.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.RegularExpressions; + +namespace DockerFileBuildHelper +{ + public class Image + { + public string User { get; private set; } + public string Name { get; private set; } + public string Tag { get; private set; } + + public string DockerHubLink + { + get + { + return User == string.Empty ? + $"https://hub.docker.com/_/{Name}" : + $"https://hub.docker.com/r/{User}/{Name}"; + } + } + + public static Image Parse(string str) + { + //${BTCPAY_IMAGE: -btcpayserver / btcpayserver:1.0.3.21} + var variableMatch = Regex.Match(str, @"\$\{[^-]+-([^\}]+)\}"); + if(variableMatch.Success) + { + str = variableMatch.Groups[1].Value; + } + Image img = new Image(); + var match = Regex.Match(str, "([^/]*/)?([^:]+):?(.*)"); + if (!match.Success) + throw new FormatException(); + img.User = match.Groups[1].Length == 0 ? string.Empty : match.Groups[1].Value.Substring(0, match.Groups[1].Value.Length - 1); + img.Name = match.Groups[2].Value; + img.Tag = match.Groups[3].Value; + if (img.Tag == string.Empty) + img.Tag = "latest"; + return img; + } + + public override string ToString() + { + StringBuilder builder = new StringBuilder(); + if (!String.IsNullOrWhiteSpace(User)) + builder.Append($"{User}/"); + builder.Append($"{Name}"); + if (!String.IsNullOrWhiteSpace(Tag)) + builder.Append($":{Tag}"); + return builder.ToString(); + } + } +} diff --git a/contrib/DockerFileBuildHelper/Program.cs b/contrib/DockerFileBuildHelper/Program.cs new file mode 100644 index 0000000..bd8cde0 --- /dev/null +++ b/contrib/DockerFileBuildHelper/Program.cs @@ -0,0 +1,271 @@ +using System; +using YamlDotNet; +using YamlDotNet.Helpers; +using System.Linq; +using System.Collections.Generic; +using System.IO; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; +using System.Net.Http; +using System.Threading.Tasks; +using System.Text; + +namespace DockerFileBuildHelper +{ + class Program + { + static int Main(string[] args) + { + string outputFile = null; + for (int i = 0; i < args.Length; i++) + { + if (args[i] == "-o") + outputFile = args[i + 1]; + } + return new Program().Run(outputFile) ? 0 : 1; + } + + private bool Run(string outputFile) + { + var fragmentDirectory = Path.GetFullPath(Path.Combine(FindRoot("contrib"), "..", "docker-compose-generator", "docker-fragments")); + List> downloading = new List>(); + List dockerInfos = new List(); + foreach(var image in new[] + { + Image.Parse("btcpayserver/docker-compose-generator"), + Image.Parse("btcpayserver/docker-compose-builder:1.23.2"), + }.Concat(GetImages(fragmentDirectory))) + { + Console.WriteLine($"Image: {image.ToString()}"); + var info = GetDockerInfo(image); + if (info == null) + { + Console.WriteLine($"Missing image info: {image}"); + return false; + } + dockerInfos.Add(info); + downloading.Add(CheckLink(info, info.DockerFilePath)); + downloading.Add(CheckLink(info, info.DockerFilePathARM32v7)); + downloading.Add(CheckLink(info, info.DockerFilePathARM64v8)); + } + + Task.WaitAll(downloading.ToArray()); + var canDownloadEverything = downloading.All(o => o.Result); + if (!canDownloadEverything) + return false; + StringBuilder builder = new StringBuilder(); + builder.AppendLine("#!/bin/bash"); + builder.AppendLine(); + builder.AppendLine("# This file is automatically generated by the DockerFileBuildHelper tool, run DockerFileBuildHelper/update-repo.sh to update it"); + builder.AppendLine("set -e"); + foreach (var info in dockerInfos) + { + builder.AppendLine($"# Build {info.Image.Name}"); + builder.AppendLine($"git clone {info.GitLink} {info.Image.Name}"); + var dockerFile = DockerFile.Parse($"{info.DockerFilePath ?? info.DockerFilePathARM32v7 ?? info.DockerFilePathARM64v8}"); + builder.AppendLine($"cd {info.Image.Name}"); + builder.AppendLine($"git checkout {info.GitRef}"); + builder.AppendLine($"cd {dockerFile.DockerFilePath}"); + builder.AppendLine($"docker build -f \"{dockerFile.DockerFileName}\" -t \"{info.Image}\" ."); + builder.AppendLine($"cd - && cd .."); + builder.AppendLine(); + builder.AppendLine(); + } + var script = builder.ToString().Replace("\r\n", "\n"); + if (string.IsNullOrEmpty(outputFile)) + outputFile = "build-all.sh"; + File.WriteAllText(outputFile, script); + Console.WriteLine($"Generated file \"{Path.GetFullPath(outputFile)}\""); + return true; + } + HttpClient client = new HttpClient(); + private async Task CheckLink(DockerInfo info, string path) + { + if (path == null) + return true; + var link = info.GetGithubLinkOf(path); + var resp = await client.GetAsync(link); + if(!resp.IsSuccessStatusCode) + { + Console.WriteLine($"\tBroken link detected for image {info.Image} ({link})"); + return false; + } + return true; + } + + private IEnumerable GetImages(string fragmentDirectory) + { + var deserializer = new DeserializerBuilder().Build(); + var serializer = new SerializerBuilder().Build(); + foreach (var file in Directory.EnumerateFiles(fragmentDirectory, "*.yml")) + { + var root = ParseDocument(file); + if (root.TryGet("services") == null) + continue; + foreach (var service in ((YamlMappingNode)root["services"]).Children) + { + var imageStr = service.Value.TryGet("image"); + if (imageStr == null) + continue; + var image = Image.Parse(imageStr.ToString()); + yield return image; + } + } + } + private DockerInfo GetDockerInfo(Image image) + { + DockerInfo dockerInfo = new DockerInfo(); + switch (image.Name) + { + case "btglnd": + dockerInfo.DockerFilePath = "BTCPayServer.Dockerfile"; + dockerInfo.GitLink = "https://github.com/vutov/lnd"; + dockerInfo.GitRef = "master"; + break; + case "docker-compose-builder": + dockerInfo.DockerFilePathARM32v7 = "linuxarm32v7.Dockerfile"; + dockerInfo.GitLink = "https://github.com/btcpayserver/docker-compose-builder"; + dockerInfo.GitRef = $"v{image.Tag}"; + break; + case "docker-compose-generator": + dockerInfo.DockerFilePath = "docker-compose-generator/linuxamd64.Dockerfile"; + dockerInfo.DockerFilePathARM32v7 = "docker-compose-generator/linuxarm32v7.Dockerfile"; + dockerInfo.GitLink = "https://github.com/btcpayserver/btcpayserver-docker"; + dockerInfo.GitRef = $"dcg-latest"; + break; + case "docker-bitcoingold": + dockerInfo.DockerFilePath = $"bitcoingold/{image.Tag}/Dockerfile"; + dockerInfo.GitLink = "https://github.com/Vutov/docker-bitcoin"; + dockerInfo.GitRef = "master"; + break; + case "clightning": + dockerInfo.DockerFilePath = $"Dockerfile"; + dockerInfo.GitLink = "https://github.com/NicolasDorier/lightning"; + dockerInfo.GitRef = $"basedon-{image.Tag}"; + break; + case "lnd": + dockerInfo.DockerFilePath = "BTCPayServer.Dockerfile"; + dockerInfo.GitLink = "https://github.com/btcpayserver/lnd"; + dockerInfo.GitRef = $"basedon-v{image.Tag}"; + break; + case "bitcoin": + dockerInfo.DockerFilePath = $"Bitcoin/{image.Tag}/linuxamd64.Dockerfile"; + dockerInfo.DockerFilePathARM32v7 = $"Bitcoin/{image.Tag}/linuxarm32v7.Dockerfile"; + dockerInfo.DockerFilePathARM64v8 = $"Bitcoin/{image.Tag}/linuxarm64v8.Dockerfile"; + dockerInfo.GitLink = "https://github.com/btcpayserver/dockerfile-deps"; + dockerInfo.GitRef = $"Bitcoin/{image.Tag}"; + break; + case "btcpayserver": + dockerInfo.DockerFilePath = "Dockerfile.linuxamd64"; + dockerInfo.DockerFilePathARM32v7 = "Dockerfile.linuxarm32v7"; + dockerInfo.GitLink = "https://github.com/btcpayserver/btcpayserver"; + dockerInfo.GitRef = $"v{image.Tag}"; + break; + case "nbxplorer": + dockerInfo.DockerFilePath = "Dockerfile.linuxamd64"; + dockerInfo.DockerFilePathARM32v7 = "Dockerfile.linuxarm32v7"; + dockerInfo.GitLink = "https://github.com/dgarage/nbxplorer"; + dockerInfo.GitRef = $"v{image.Tag}"; + break; + case "dogecoin": + dockerInfo.DockerFilePath = $"dogecoin/{image.Tag}/Dockerfile"; + dockerInfo.GitLink = "https://github.com/rockstardev/docker-bitcoin"; + dockerInfo.GitRef = "feature/dogecoin"; + break; + case "docker-feathercoin": + dockerInfo.DockerFilePath = $"feathercoin/{image.Tag}/Dockerfile"; + dockerInfo.GitLink = "https://github.com/ChekaZ/docker"; + dockerInfo.GitRef = "master"; + break; + case "docker-groestlcoin": + dockerInfo.DockerFilePath = $"groestlcoin/{image.Tag}/Dockerfile"; + dockerInfo.GitLink = "https://github.com/NicolasDorier/docker-bitcoin"; + dockerInfo.GitRef = "master"; + break; + case "docker-viacoin": + dockerInfo.DockerFilePath = $"viacoin/{image.Tag}/docker-viacoin"; + dockerInfo.GitLink = "https://github.com/viacoin/docker-viacoin"; + dockerInfo.GitRef = "master"; + break; + case "docker-litecoin": + dockerInfo.DockerFilePath = $"litecoin/{image.Tag}/Dockerfile"; + dockerInfo.GitLink = "https://github.com/NicolasDorier/docker-bitcoin"; + dockerInfo.GitRef = "master"; + break; + case "docker-monacoin": + dockerInfo.DockerFilePath = $"monacoin/{image.Tag}/Dockerfile"; + dockerInfo.GitLink = "https://github.com/wakiyamap/docker-bitcoin"; + dockerInfo.GitRef = "master"; + break; + case "nginx": + dockerInfo.DockerFilePath = $"stable/stretch/Dockerfile"; + dockerInfo.GitLink = "https://github.com/nginxinc/docker-nginx"; + dockerInfo.GitRef = $"master"; + break; + case "docker-gen": + dockerInfo.DockerFilePath = $"linuxamd64.Dockerfile"; + dockerInfo.DockerFilePathARM32v7 = $"linuxarm32v7.Dockerfile"; + dockerInfo.GitLink = "https://github.com/btcpayserver/docker-gen"; + dockerInfo.GitRef = $"v{image.Tag}"; + break; + case "letsencrypt-nginx-proxy-companion": + dockerInfo.DockerFilePath = $"linuxamd64.Dockerfile"; + dockerInfo.DockerFilePathARM32v7 = $"linuxarm32v7.Dockerfile"; + dockerInfo.GitLink = "https://github.com/btcpayserver/docker-letsencrypt-nginx-proxy-companion"; + dockerInfo.GitRef = $"v{image.Tag}"; + break; + case "postgres": + dockerInfo.DockerFilePath = $"9.6/Dockerfile"; + dockerInfo.DockerFilePathARM32v7 = $"9.6/Dockerfile"; + dockerInfo.GitLink = "https://github.com/docker-library/postgres"; + dockerInfo.GitRef = $"b7cb3c6eacea93be2259381033be3cc435649369"; + break; + case "traefik": + dockerInfo.DockerFilePath = $"scratch/amd64/Dockerfile"; + dockerInfo.DockerFilePathARM32v7 = $"scratch/arm/Dockerfile"; + dockerInfo.GitLink = "https://github.com/containous/traefik-library-image"; + dockerInfo.GitRef = $"master"; + break; + default: + return null; + } + dockerInfo.DockerHubLink = image.DockerHubLink; + dockerInfo.Image = image; + return dockerInfo; + } + + private YamlMappingNode ParseDocument(string fragment) + { + var input = new StringReader(File.ReadAllText(fragment)); + YamlStream stream = new YamlStream(); + stream.Load(input); + return (YamlMappingNode)stream.Documents[0].RootNode; + } + + private static void DeleteDirectory(string outputDirectory) + { + try + { + Directory.Delete(outputDirectory, true); + } + catch + { + } + } + + private static string FindRoot(string rootDirectory) + { + string directory = Directory.GetCurrentDirectory(); + int i = 0; + while (true) + { + if (i > 10) + throw new DirectoryNotFoundException(rootDirectory); + if (directory.EndsWith(rootDirectory)) + return directory; + directory = Path.GetFullPath(Path.Combine(directory, "..")); + i++; + } + } + } +} diff --git a/contrib/DockerFileBuildHelper/README.md b/contrib/DockerFileBuildHelper/README.md new file mode 100644 index 0000000..f1b1d43 --- /dev/null +++ b/contrib/DockerFileBuildHelper/README.md @@ -0,0 +1,16 @@ +# DockerFile build helper + +By default, when you use docker deployment, you are fetching pre-built docker images hosted on dockerhub. +While this bring the advantage that deployment is fast and reliable, this also mean that you are ultimately trusting the owner of the docker images. +This repository generate a script that you can use to build all images from the sources by yourself. + +## How to use? + +Install [.NET Core SDK](https://dotnet.microsoft.com/download) and run: + +```bash +./run.sh +``` + +This will build a `build-all.sh` file. +Note that the diff --git a/contrib/DockerFileBuildHelper/run.sh b/contrib/DockerFileBuildHelper/run.sh new file mode 100755 index 0000000..c2bba29 --- /dev/null +++ b/contrib/DockerFileBuildHelper/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +dotnet run --no-launch-profile -c Release -- $@ \ No newline at end of file diff --git a/contrib/DockerFileBuildHelper/update-repo.ps1 b/contrib/DockerFileBuildHelper/update-repo.ps1 new file mode 100755 index 0000000..d90ded0 --- /dev/null +++ b/contrib/DockerFileBuildHelper/update-repo.ps1 @@ -0,0 +1 @@ +dotnet run --no-launch-profile -c Release -- -o "../build-all-images.sh" \ No newline at end of file diff --git a/contrib/DockerFileBuildHelper/update-repo.sh b/contrib/DockerFileBuildHelper/update-repo.sh new file mode 100755 index 0000000..c50753c --- /dev/null +++ b/contrib/DockerFileBuildHelper/update-repo.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +dotnet run --no-launch-profile -c Release -- -o "../build-all-images.sh" \ No newline at end of file diff --git a/contrib/build-all-images.sh b/contrib/build-all-images.sh new file mode 100755 index 0000000..0877848 --- /dev/null +++ b/contrib/build-all-images.sh @@ -0,0 +1,202 @@ +#!/bin/bash + +# This file is automatically generated by the DockerFileBuildHelper tool, run DockerFileBuildHelper/update-repo.sh to update it +set -e +# Build docker-compose-generator +git clone https://github.com/btcpayserver/btcpayserver-docker docker-compose-generator +cd docker-compose-generator +git checkout dcg-latest +cd docker-compose-generator +docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/docker-compose-generator:latest" . +cd - && cd .. + + +# Build docker-compose-builder +git clone https://github.com/btcpayserver/docker-compose-builder docker-compose-builder +cd docker-compose-builder +git checkout v1.23.2 +cd . +docker build -f "linuxarm32v7.Dockerfile" -t "btcpayserver/docker-compose-builder:1.23.2" . +cd - && cd .. + + +# Build btglnd +git clone https://github.com/vutov/lnd btglnd +cd btglnd +git checkout master +cd . +docker build -f "BTCPayServer.Dockerfile" -t "kamigawabul/btglnd:latest" . +cd - && cd .. + + +# Build docker-bitcoingold +git clone https://github.com/Vutov/docker-bitcoin docker-bitcoingold +cd docker-bitcoingold +git checkout master +cd bitcoingold/0.15.2 +docker build -f "Dockerfile" -t "kamigawabul/docker-bitcoingold:0.15.2" . +cd - && cd .. + + +# Build clightning +git clone https://github.com/NicolasDorier/lightning clightning +cd clightning +git checkout basedon-v0.6.2-3 +cd . +docker build -f "Dockerfile" -t "nicolasdorier/clightning:v0.6.2-3" . +cd - && cd .. + + +# Build lnd +git clone https://github.com/btcpayserver/lnd lnd +cd lnd +git checkout basedon-v0.5-beta-2 +cd . +docker build -f "BTCPayServer.Dockerfile" -t "btcpayserver/lnd:0.5-beta-2" . +cd - && cd .. + + +# Build bitcoin +git clone https://github.com/btcpayserver/dockerfile-deps bitcoin +cd bitcoin +git checkout Bitcoin/0.17.0 +cd Bitcoin/0.17.0 +docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/bitcoin:0.17.0" . +cd - && cd .. + + +# Build btcpayserver +git clone https://github.com/btcpayserver/btcpayserver btcpayserver +cd btcpayserver +git checkout v1.0.3.21 +cd . +docker build -f "Dockerfile.linuxamd64" -t "btcpayserver/btcpayserver:1.0.3.21" . +cd - && cd .. + + +# Build dogecoin +git clone https://github.com/rockstardev/docker-bitcoin dogecoin +cd dogecoin +git checkout feature/dogecoin +cd dogecoin/1.10.0 +docker build -f "Dockerfile" -t "rockstardev/dogecoin:1.10.0" . +cd - && cd .. + + +# Build docker-feathercoin +git clone https://github.com/ChekaZ/docker docker-feathercoin +cd docker-feathercoin +git checkout master +cd feathercoin/0.16.3 +docker build -f "Dockerfile" -t "chekaz/docker-feathercoin:0.16.3" . +cd - && cd .. + + +# Build docker-groestlcoin +git clone https://github.com/NicolasDorier/docker-bitcoin docker-groestlcoin +cd docker-groestlcoin +git checkout master +cd groestlcoin/2.16.3 +docker build -f "Dockerfile" -t "nicolasdorier/docker-groestlcoin:2.16.3" . +cd - && cd .. + + +# Build clightning +git clone https://github.com/NicolasDorier/lightning clightning +cd clightning +git checkout basedon-v0.6.2-3 +cd . +docker build -f "Dockerfile" -t "nicolasdorier/clightning:v0.6.2-3" . +cd - && cd .. + + +# Build lnd +git clone https://github.com/btcpayserver/lnd lnd +cd lnd +git checkout basedon-v0.5-beta-2 +cd . +docker build -f "BTCPayServer.Dockerfile" -t "btcpayserver/lnd:0.5-beta-2" . +cd - && cd .. + + +# Build docker-litecoin +git clone https://github.com/NicolasDorier/docker-bitcoin docker-litecoin +cd docker-litecoin +git checkout master +cd litecoin/0.16.3 +docker build -f "Dockerfile" -t "nicolasdorier/docker-litecoin:0.16.3" . +cd - && cd .. + + +# Build docker-monacoin +git clone https://github.com/wakiyamap/docker-bitcoin docker-monacoin +cd docker-monacoin +git checkout master +cd monacoin/0.16.3 +docker build -f "Dockerfile" -t "wakiyamap/docker-monacoin:0.16.3" . +cd - && cd .. + + +# Build nbxplorer +git clone https://github.com/dgarage/nbxplorer nbxplorer +cd nbxplorer +git checkout v2.0.0.1 +cd . +docker build -f "Dockerfile.linuxamd64" -t "nicolasdorier/nbxplorer:2.0.0.1" . +cd - && cd .. + + +# Build nginx +git clone https://github.com/nginxinc/docker-nginx nginx +cd nginx +git checkout master +cd stable/stretch +docker build -f "Dockerfile" -t "nginx:stable" . +cd - && cd .. + + +# Build docker-gen +git clone https://github.com/btcpayserver/docker-gen docker-gen +cd docker-gen +git checkout v0.7.4 +cd . +docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/docker-gen:0.7.4" . +cd - && cd .. + + +# Build letsencrypt-nginx-proxy-companion +git clone https://github.com/btcpayserver/docker-letsencrypt-nginx-proxy-companion letsencrypt-nginx-proxy-companion +cd letsencrypt-nginx-proxy-companion +git checkout v1.10.0 +cd . +docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/letsencrypt-nginx-proxy-companion:1.10.0" . +cd - && cd .. + + +# Build postgres +git clone https://github.com/docker-library/postgres postgres +cd postgres +git checkout b7cb3c6eacea93be2259381033be3cc435649369 +cd 9.6 +docker build -f "Dockerfile" -t "postgres:9.6.5" . +cd - && cd .. + + +# Build traefik +git clone https://github.com/containous/traefik-library-image traefik +cd traefik +git checkout master +cd scratch/amd64 +docker build -f "Dockerfile" -t "traefik:latest" . +cd - && cd .. + + +# Build docker-viacoin +git clone https://github.com/viacoin/docker-viacoin docker-viacoin +cd docker-viacoin +git checkout master +cd viacoin/0.15.2 +docker build -f "docker-viacoin" -t "romanornr/docker-viacoin:0.15.2" . +cd - && cd .. + + From c84ab3f4fcd2d011746c1dfbb79d99da2cd0c127 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Sun, 2 Dec 2018 17:06:36 +0900 Subject: [PATCH 32/36] Make build-all-images work on arm --- contrib/DockerFileBuildHelper/DockerFile.cs | 16 +- contrib/DockerFileBuildHelper/Program.cs | 45 ++++- .../DockerFileBuildHelper/StringBuilderEx.cs | 45 +++++ contrib/build-all-images.sh | 186 +++++++++++++----- 4 files changed, 237 insertions(+), 55 deletions(-) create mode 100644 contrib/DockerFileBuildHelper/StringBuilderEx.cs diff --git a/contrib/DockerFileBuildHelper/DockerFile.cs b/contrib/DockerFileBuildHelper/DockerFile.cs index b1d7d82..ff55dbb 100644 --- a/contrib/DockerFileBuildHelper/DockerFile.cs +++ b/contrib/DockerFileBuildHelper/DockerFile.cs @@ -7,7 +7,17 @@ namespace DockerFileBuildHelper public class DockerFile { public string DockerFileName { get; private set; } - public string DockerFilePath { get; private set; } + public string DockerDirectoryPath { get; private set; } + public string DockerFullPath + { + get + { + if (DockerDirectoryPath == ".") + return $"{DockerFileName}"; + else + return $"{DockerDirectoryPath}/{DockerFileName}"; + } + } public static DockerFile Parse(string str) { @@ -16,11 +26,11 @@ namespace DockerFileBuildHelper file.DockerFileName = str.Substring(lastPart + 1); if (lastPart == -1) { - file.DockerFilePath = "."; + file.DockerDirectoryPath = "."; } else { - file.DockerFilePath = str.Substring(0, lastPart); + file.DockerDirectoryPath = str.Substring(0, lastPart); } return file; } diff --git a/contrib/DockerFileBuildHelper/Program.cs b/contrib/DockerFileBuildHelper/Program.cs index bd8cde0..2f1a583 100644 --- a/contrib/DockerFileBuildHelper/Program.cs +++ b/contrib/DockerFileBuildHelper/Program.cs @@ -53,21 +53,58 @@ namespace DockerFileBuildHelper var canDownloadEverything = downloading.All(o => o.Result); if (!canDownloadEverything) return false; - StringBuilder builder = new StringBuilder(); + var builder = new StringBuilderEx(); builder.AppendLine("#!/bin/bash"); builder.AppendLine(); builder.AppendLine("# This file is automatically generated by the DockerFileBuildHelper tool, run DockerFileBuildHelper/update-repo.sh to update it"); builder.AppendLine("set -e"); + builder.AppendLine("DOCKERFILE=\"\""); + builder.AppendLine(); + builder.AppendLine(); foreach (var info in dockerInfos) { builder.AppendLine($"# Build {info.Image.Name}"); + bool mightBeUnavailable = false; + if (info.DockerFilePath != null) + { + var dockerFile = DockerFile.Parse(info.DockerFilePath); + builder.AppendLine($"# {info.GetGithubLinkOf(dockerFile.DockerFullPath)}"); + builder.AppendLine($"DOCKERFILE=\"{dockerFile.DockerFullPath}\""); + } + else + { + builder.AppendLine($"DOCKERFILE=\"\""); + mightBeUnavailable = true; + } + if (info.DockerFilePathARM32v7 != null) + { + var dockerFile = DockerFile.Parse(info.DockerFilePathARM32v7); + builder.AppendLine($"# {info.GetGithubLinkOf(dockerFile.DockerFullPath)}"); + builder.AppendLine($"[[ \"$(uname -m)\" == \"armv7l\" ]] && DOCKERFILE=\"{dockerFile.DockerFullPath}\""); + } + if (info.DockerFilePathARM64v8 != null) + { + var dockerFile = DockerFile.Parse(info.DockerFilePathARM64v8); + builder.AppendLine($"# {info.GetGithubLinkOf(dockerFile.DockerFullPath)}"); + builder.AppendLine($"[[ \"$(uname -m)\" == \"aarch64\" ]] && DOCKERFILE=\"{dockerFile.DockerFullPath}\""); + } + if(mightBeUnavailable) + { + builder.AppendLine($"if [[ \"$DOCKERFILE\" ]]; then"); + builder.Indent++; + } + builder.AppendLine($"echo \"Building {info.Image.ToString()}\""); builder.AppendLine($"git clone {info.GitLink} {info.Image.Name}"); - var dockerFile = DockerFile.Parse($"{info.DockerFilePath ?? info.DockerFilePathARM32v7 ?? info.DockerFilePathARM64v8}"); builder.AppendLine($"cd {info.Image.Name}"); builder.AppendLine($"git checkout {info.GitRef}"); - builder.AppendLine($"cd {dockerFile.DockerFilePath}"); - builder.AppendLine($"docker build -f \"{dockerFile.DockerFileName}\" -t \"{info.Image}\" ."); + builder.AppendLine($"cd \"$(dirname $DOCKERFILE)\""); + builder.AppendLine($"docker build -f \"$DOCKERFILE\" -t \"{info.Image}\" ."); builder.AppendLine($"cd - && cd .."); + if (mightBeUnavailable) + { + builder.Indent--; + builder.AppendLine($"fi"); + } builder.AppendLine(); builder.AppendLine(); } diff --git a/contrib/DockerFileBuildHelper/StringBuilderEx.cs b/contrib/DockerFileBuildHelper/StringBuilderEx.cs new file mode 100644 index 0000000..6823e58 --- /dev/null +++ b/contrib/DockerFileBuildHelper/StringBuilderEx.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace DockerFileBuildHelper +{ + public class StringBuilderEx + { + StringBuilder _Builder = new StringBuilder(); + public StringBuilderEx() + { + + } + + public int Indent { get; set; } + + public void Append(string str) + { + _Builder.Append(GetIndents()); + _Builder.Append(str); + } + + private string GetIndents() + { + return new String(Enumerable.Range(0, Indent).Select(_ => '\t').ToArray()); + } + + public void AppendLine(string str) + { + _Builder.Append(GetIndents()); + _Builder.AppendLine(str); + } + + public override string ToString() + { + return _Builder.ToString(); + } + + internal void AppendLine() + { + _Builder.AppendLine(); + } + } +} diff --git a/contrib/build-all-images.sh b/contrib/build-all-images.sh index 0877848..f740aea 100755 --- a/contrib/build-all-images.sh +++ b/contrib/build-all-images.sh @@ -2,201 +2,291 @@ # This file is automatically generated by the DockerFileBuildHelper tool, run DockerFileBuildHelper/update-repo.sh to update it set -e +DOCKERFILE="" + + # Build docker-compose-generator +# https://raw.githubusercontent.com/btcpayserver/btcpayserver-docker/dcg-latest/docker-compose-generator/linuxamd64.Dockerfile +DOCKERFILE="docker-compose-generator/linuxamd64.Dockerfile" +# https://raw.githubusercontent.com/btcpayserver/btcpayserver-docker/dcg-latest/docker-compose-generator/linuxarm32v7.Dockerfile +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="docker-compose-generator/linuxarm32v7.Dockerfile" +echo "Building btcpayserver/docker-compose-generator:latest" git clone https://github.com/btcpayserver/btcpayserver-docker docker-compose-generator cd docker-compose-generator git checkout dcg-latest -cd docker-compose-generator -docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/docker-compose-generator:latest" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "btcpayserver/docker-compose-generator:latest" . cd - && cd .. # Build docker-compose-builder -git clone https://github.com/btcpayserver/docker-compose-builder docker-compose-builder -cd docker-compose-builder -git checkout v1.23.2 -cd . -docker build -f "linuxarm32v7.Dockerfile" -t "btcpayserver/docker-compose-builder:1.23.2" . -cd - && cd .. +DOCKERFILE="" +# https://raw.githubusercontent.com/btcpayserver/docker-compose-builder/v1.23.2/linuxarm32v7.Dockerfile +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="linuxarm32v7.Dockerfile" +if [[ "$DOCKERFILE" ]]; then + echo "Building btcpayserver/docker-compose-builder:1.23.2" + git clone https://github.com/btcpayserver/docker-compose-builder docker-compose-builder + cd docker-compose-builder + git checkout v1.23.2 + cd "$(dirname $DOCKERFILE)" + docker build -f "$DOCKERFILE" -t "btcpayserver/docker-compose-builder:1.23.2" . + cd - && cd .. +fi # Build btglnd +# https://raw.githubusercontent.com/vutov/lnd/master/BTCPayServer.Dockerfile +DOCKERFILE="BTCPayServer.Dockerfile" +echo "Building kamigawabul/btglnd:latest" git clone https://github.com/vutov/lnd btglnd cd btglnd git checkout master -cd . -docker build -f "BTCPayServer.Dockerfile" -t "kamigawabul/btglnd:latest" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "kamigawabul/btglnd:latest" . cd - && cd .. # Build docker-bitcoingold +# https://raw.githubusercontent.com/Vutov/docker-bitcoin/master/bitcoingold/0.15.2/Dockerfile +DOCKERFILE="bitcoingold/0.15.2/Dockerfile" +echo "Building kamigawabul/docker-bitcoingold:0.15.2" git clone https://github.com/Vutov/docker-bitcoin docker-bitcoingold cd docker-bitcoingold git checkout master -cd bitcoingold/0.15.2 -docker build -f "Dockerfile" -t "kamigawabul/docker-bitcoingold:0.15.2" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "kamigawabul/docker-bitcoingold:0.15.2" . cd - && cd .. # Build clightning +# https://raw.githubusercontent.com/NicolasDorier/lightning/basedon-v0.6.2-3/Dockerfile +DOCKERFILE="Dockerfile" +echo "Building nicolasdorier/clightning:v0.6.2-3" git clone https://github.com/NicolasDorier/lightning clightning cd clightning git checkout basedon-v0.6.2-3 -cd . -docker build -f "Dockerfile" -t "nicolasdorier/clightning:v0.6.2-3" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "nicolasdorier/clightning:v0.6.2-3" . cd - && cd .. # Build lnd +# https://raw.githubusercontent.com/btcpayserver/lnd/basedon-v0.5-beta-2/BTCPayServer.Dockerfile +DOCKERFILE="BTCPayServer.Dockerfile" +echo "Building btcpayserver/lnd:0.5-beta-2" git clone https://github.com/btcpayserver/lnd lnd cd lnd git checkout basedon-v0.5-beta-2 -cd . -docker build -f "BTCPayServer.Dockerfile" -t "btcpayserver/lnd:0.5-beta-2" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "btcpayserver/lnd:0.5-beta-2" . cd - && cd .. # Build bitcoin +# https://raw.githubusercontent.com/btcpayserver/dockerfile-deps/Bitcoin/0.17.0/Bitcoin/0.17.0/linuxamd64.Dockerfile +DOCKERFILE="Bitcoin/0.17.0/linuxamd64.Dockerfile" +# https://raw.githubusercontent.com/btcpayserver/dockerfile-deps/Bitcoin/0.17.0/Bitcoin/0.17.0/linuxarm32v7.Dockerfile +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="Bitcoin/0.17.0/linuxarm32v7.Dockerfile" +# https://raw.githubusercontent.com/btcpayserver/dockerfile-deps/Bitcoin/0.17.0/Bitcoin/0.17.0/linuxarm64v8.Dockerfile +[[ "$(uname -m)" == "aarch64" ]] && DOCKERFILE="Bitcoin/0.17.0/linuxarm64v8.Dockerfile" +echo "Building btcpayserver/bitcoin:0.17.0" git clone https://github.com/btcpayserver/dockerfile-deps bitcoin cd bitcoin git checkout Bitcoin/0.17.0 -cd Bitcoin/0.17.0 -docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/bitcoin:0.17.0" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "btcpayserver/bitcoin:0.17.0" . cd - && cd .. # Build btcpayserver +# https://raw.githubusercontent.com/btcpayserver/btcpayserver/v1.0.3.21/Dockerfile.linuxamd64 +DOCKERFILE="Dockerfile.linuxamd64" +# https://raw.githubusercontent.com/btcpayserver/btcpayserver/v1.0.3.21/Dockerfile.linuxarm32v7 +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="Dockerfile.linuxarm32v7" +echo "Building btcpayserver/btcpayserver:1.0.3.21" git clone https://github.com/btcpayserver/btcpayserver btcpayserver cd btcpayserver git checkout v1.0.3.21 -cd . -docker build -f "Dockerfile.linuxamd64" -t "btcpayserver/btcpayserver:1.0.3.21" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "btcpayserver/btcpayserver:1.0.3.21" . cd - && cd .. # Build dogecoin +# https://raw.githubusercontent.com/rockstardev/docker-bitcoin/feature/dogecoin/dogecoin/1.10.0/Dockerfile +DOCKERFILE="dogecoin/1.10.0/Dockerfile" +echo "Building rockstardev/dogecoin:1.10.0" git clone https://github.com/rockstardev/docker-bitcoin dogecoin cd dogecoin git checkout feature/dogecoin -cd dogecoin/1.10.0 -docker build -f "Dockerfile" -t "rockstardev/dogecoin:1.10.0" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "rockstardev/dogecoin:1.10.0" . cd - && cd .. # Build docker-feathercoin +# https://raw.githubusercontent.com/ChekaZ/docker/master/feathercoin/0.16.3/Dockerfile +DOCKERFILE="feathercoin/0.16.3/Dockerfile" +echo "Building chekaz/docker-feathercoin:0.16.3" git clone https://github.com/ChekaZ/docker docker-feathercoin cd docker-feathercoin git checkout master -cd feathercoin/0.16.3 -docker build -f "Dockerfile" -t "chekaz/docker-feathercoin:0.16.3" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "chekaz/docker-feathercoin:0.16.3" . cd - && cd .. # Build docker-groestlcoin +# https://raw.githubusercontent.com/NicolasDorier/docker-bitcoin/master/groestlcoin/2.16.3/Dockerfile +DOCKERFILE="groestlcoin/2.16.3/Dockerfile" +echo "Building nicolasdorier/docker-groestlcoin:2.16.3" git clone https://github.com/NicolasDorier/docker-bitcoin docker-groestlcoin cd docker-groestlcoin git checkout master -cd groestlcoin/2.16.3 -docker build -f "Dockerfile" -t "nicolasdorier/docker-groestlcoin:2.16.3" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "nicolasdorier/docker-groestlcoin:2.16.3" . cd - && cd .. # Build clightning +# https://raw.githubusercontent.com/NicolasDorier/lightning/basedon-v0.6.2-3/Dockerfile +DOCKERFILE="Dockerfile" +echo "Building nicolasdorier/clightning:v0.6.2-3" git clone https://github.com/NicolasDorier/lightning clightning cd clightning git checkout basedon-v0.6.2-3 -cd . -docker build -f "Dockerfile" -t "nicolasdorier/clightning:v0.6.2-3" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "nicolasdorier/clightning:v0.6.2-3" . cd - && cd .. # Build lnd +# https://raw.githubusercontent.com/btcpayserver/lnd/basedon-v0.5-beta-2/BTCPayServer.Dockerfile +DOCKERFILE="BTCPayServer.Dockerfile" +echo "Building btcpayserver/lnd:0.5-beta-2" git clone https://github.com/btcpayserver/lnd lnd cd lnd git checkout basedon-v0.5-beta-2 -cd . -docker build -f "BTCPayServer.Dockerfile" -t "btcpayserver/lnd:0.5-beta-2" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "btcpayserver/lnd:0.5-beta-2" . cd - && cd .. # Build docker-litecoin +# https://raw.githubusercontent.com/NicolasDorier/docker-bitcoin/master/litecoin/0.16.3/Dockerfile +DOCKERFILE="litecoin/0.16.3/Dockerfile" +echo "Building nicolasdorier/docker-litecoin:0.16.3" git clone https://github.com/NicolasDorier/docker-bitcoin docker-litecoin cd docker-litecoin git checkout master -cd litecoin/0.16.3 -docker build -f "Dockerfile" -t "nicolasdorier/docker-litecoin:0.16.3" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "nicolasdorier/docker-litecoin:0.16.3" . cd - && cd .. # Build docker-monacoin +# https://raw.githubusercontent.com/wakiyamap/docker-bitcoin/master/monacoin/0.16.3/Dockerfile +DOCKERFILE="monacoin/0.16.3/Dockerfile" +echo "Building wakiyamap/docker-monacoin:0.16.3" git clone https://github.com/wakiyamap/docker-bitcoin docker-monacoin cd docker-monacoin git checkout master -cd monacoin/0.16.3 -docker build -f "Dockerfile" -t "wakiyamap/docker-monacoin:0.16.3" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "wakiyamap/docker-monacoin:0.16.3" . cd - && cd .. # Build nbxplorer +# https://raw.githubusercontent.com/dgarage/nbxplorer/v2.0.0.1/Dockerfile.linuxamd64 +DOCKERFILE="Dockerfile.linuxamd64" +# https://raw.githubusercontent.com/dgarage/nbxplorer/v2.0.0.1/Dockerfile.linuxarm32v7 +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="Dockerfile.linuxarm32v7" +echo "Building nicolasdorier/nbxplorer:2.0.0.1" git clone https://github.com/dgarage/nbxplorer nbxplorer cd nbxplorer git checkout v2.0.0.1 -cd . -docker build -f "Dockerfile.linuxamd64" -t "nicolasdorier/nbxplorer:2.0.0.1" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "nicolasdorier/nbxplorer:2.0.0.1" . cd - && cd .. # Build nginx +# https://raw.githubusercontent.com/nginxinc/docker-nginx/master/stable/stretch/Dockerfile +DOCKERFILE="stable/stretch/Dockerfile" +echo "Building nginx:stable" git clone https://github.com/nginxinc/docker-nginx nginx cd nginx git checkout master -cd stable/stretch -docker build -f "Dockerfile" -t "nginx:stable" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "nginx:stable" . cd - && cd .. # Build docker-gen +# https://raw.githubusercontent.com/btcpayserver/docker-gen/v0.7.4/linuxamd64.Dockerfile +DOCKERFILE="linuxamd64.Dockerfile" +# https://raw.githubusercontent.com/btcpayserver/docker-gen/v0.7.4/linuxarm32v7.Dockerfile +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="linuxarm32v7.Dockerfile" +echo "Building btcpayserver/docker-gen:0.7.4" git clone https://github.com/btcpayserver/docker-gen docker-gen cd docker-gen git checkout v0.7.4 -cd . -docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/docker-gen:0.7.4" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "btcpayserver/docker-gen:0.7.4" . cd - && cd .. # Build letsencrypt-nginx-proxy-companion +# https://raw.githubusercontent.com/btcpayserver/docker-letsencrypt-nginx-proxy-companion/v1.10.0/linuxamd64.Dockerfile +DOCKERFILE="linuxamd64.Dockerfile" +# https://raw.githubusercontent.com/btcpayserver/docker-letsencrypt-nginx-proxy-companion/v1.10.0/linuxarm32v7.Dockerfile +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="linuxarm32v7.Dockerfile" +echo "Building btcpayserver/letsencrypt-nginx-proxy-companion:1.10.0" git clone https://github.com/btcpayserver/docker-letsencrypt-nginx-proxy-companion letsencrypt-nginx-proxy-companion cd letsencrypt-nginx-proxy-companion git checkout v1.10.0 -cd . -docker build -f "linuxamd64.Dockerfile" -t "btcpayserver/letsencrypt-nginx-proxy-companion:1.10.0" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "btcpayserver/letsencrypt-nginx-proxy-companion:1.10.0" . cd - && cd .. # Build postgres +# https://raw.githubusercontent.com/docker-library/postgres/b7cb3c6eacea93be2259381033be3cc435649369/9.6/Dockerfile +DOCKERFILE="9.6/Dockerfile" +# https://raw.githubusercontent.com/docker-library/postgres/b7cb3c6eacea93be2259381033be3cc435649369/9.6/Dockerfile +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="9.6/Dockerfile" +echo "Building postgres:9.6.5" git clone https://github.com/docker-library/postgres postgres cd postgres git checkout b7cb3c6eacea93be2259381033be3cc435649369 -cd 9.6 -docker build -f "Dockerfile" -t "postgres:9.6.5" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "postgres:9.6.5" . cd - && cd .. # Build traefik +# https://raw.githubusercontent.com/containous/traefik-library-image/master/scratch/amd64/Dockerfile +DOCKERFILE="scratch/amd64/Dockerfile" +# https://raw.githubusercontent.com/containous/traefik-library-image/master/scratch/arm/Dockerfile +[[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="scratch/arm/Dockerfile" +echo "Building traefik:latest" git clone https://github.com/containous/traefik-library-image traefik cd traefik git checkout master -cd scratch/amd64 -docker build -f "Dockerfile" -t "traefik:latest" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "traefik:latest" . cd - && cd .. # Build docker-viacoin +# https://raw.githubusercontent.com/viacoin/docker-viacoin/master/viacoin/0.15.2/docker-viacoin +DOCKERFILE="viacoin/0.15.2/docker-viacoin" +echo "Building romanornr/docker-viacoin:0.15.2" git clone https://github.com/viacoin/docker-viacoin docker-viacoin cd docker-viacoin git checkout master -cd viacoin/0.15.2 -docker build -f "docker-viacoin" -t "romanornr/docker-viacoin:0.15.2" . +cd "$(dirname $DOCKERFILE)" +docker build -f "$DOCKERFILE" -t "romanornr/docker-viacoin:0.15.2" . cd - && cd .. From eb8269156a4c62fc4b500a7fc8b9b27aa957dd17 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 4 Dec 2018 00:00:12 +0900 Subject: [PATCH 33/36] bump --- docker-compose-generator/docker-fragments/btcpayserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-generator/docker-fragments/btcpayserver.yml b/docker-compose-generator/docker-fragments/btcpayserver.yml index 9a17bfc..5bd62e9 100644 --- a/docker-compose-generator/docker-fragments/btcpayserver.yml +++ b/docker-compose-generator/docker-fragments/btcpayserver.yml @@ -4,7 +4,7 @@ services: btcpayserver: restart: unless-stopped - image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.21} + image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.22} expose: - "49392" environment: From 7b906d900fd264e1325df1921d91b23c5c250d2c Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 4 Dec 2018 13:06:20 +0900 Subject: [PATCH 34/36] bump --- docker-compose-generator/docker-fragments/btcpayserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-generator/docker-fragments/btcpayserver.yml b/docker-compose-generator/docker-fragments/btcpayserver.yml index 5bd62e9..9309a8b 100644 --- a/docker-compose-generator/docker-fragments/btcpayserver.yml +++ b/docker-compose-generator/docker-fragments/btcpayserver.yml @@ -4,7 +4,7 @@ services: btcpayserver: restart: unless-stopped - image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.22} + image: ${BTCPAY_IMAGE:-btcpayserver/btcpayserver:1.0.3.23} expose: - "49392" environment: From 84db61bfbac48f6c4864dec98a4bb63746fff68b Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 4 Dec 2018 13:51:21 +0900 Subject: [PATCH 35/36] Add Dash to build-all-images.sh --- contrib/DockerFileBuildHelper/Program.cs | 7 +++++++ contrib/build-all-images.sh | 10 +++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/contrib/DockerFileBuildHelper/Program.cs b/contrib/DockerFileBuildHelper/Program.cs index 2f1a583..c9b4bbf 100644 --- a/contrib/DockerFileBuildHelper/Program.cs +++ b/contrib/DockerFileBuildHelper/Program.cs @@ -192,6 +192,13 @@ namespace DockerFileBuildHelper dockerInfo.GitLink = "https://github.com/btcpayserver/dockerfile-deps"; dockerInfo.GitRef = $"Bitcoin/{image.Tag}"; break; + case "dash": + dockerInfo.DockerFilePath = $"Dash/{image.Tag}/linuxamd64.Dockerfile"; + dockerInfo.DockerFilePathARM32v7 = $"Dash/{image.Tag}/linuxarm32v7.Dockerfile"; + dockerInfo.DockerFilePathARM64v8 = $"Dash/{image.Tag}/linuxarm64v8.Dockerfile"; + dockerInfo.GitLink = "https://github.com/btcpayserver/dockerfile-deps"; + dockerInfo.GitRef = $"Dash/{image.Tag}"; + break; case "btcpayserver": dockerInfo.DockerFilePath = "Dockerfile.linuxamd64"; dockerInfo.DockerFilePathARM32v7 = "Dockerfile.linuxarm32v7"; diff --git a/contrib/build-all-images.sh b/contrib/build-all-images.sh index f740aea..ea0818e 100755 --- a/contrib/build-all-images.sh +++ b/contrib/build-all-images.sh @@ -99,16 +99,16 @@ cd - && cd .. # Build btcpayserver -# https://raw.githubusercontent.com/btcpayserver/btcpayserver/v1.0.3.21/Dockerfile.linuxamd64 +# https://raw.githubusercontent.com/btcpayserver/btcpayserver/v1.0.3.23/Dockerfile.linuxamd64 DOCKERFILE="Dockerfile.linuxamd64" -# https://raw.githubusercontent.com/btcpayserver/btcpayserver/v1.0.3.21/Dockerfile.linuxarm32v7 +# https://raw.githubusercontent.com/btcpayserver/btcpayserver/v1.0.3.23/Dockerfile.linuxarm32v7 [[ "$(uname -m)" == "armv7l" ]] && DOCKERFILE="Dockerfile.linuxarm32v7" -echo "Building btcpayserver/btcpayserver:1.0.3.21" +echo "Building btcpayserver/btcpayserver:1.0.3.23" git clone https://github.com/btcpayserver/btcpayserver btcpayserver cd btcpayserver -git checkout v1.0.3.21 +git checkout v1.0.3.23 cd "$(dirname $DOCKERFILE)" -docker build -f "$DOCKERFILE" -t "btcpayserver/btcpayserver:1.0.3.21" . +docker build -f "$DOCKERFILE" -t "btcpayserver/btcpayserver:1.0.3.23" . cd - && cd .. From fb0e2a10a1098a8a2a8218e13997213a7ffe51cb Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Tue, 4 Dec 2018 14:01:03 +0900 Subject: [PATCH 36/36] add dash to list of coin --- btcpay-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btcpay-setup.sh b/btcpay-setup.sh index 3552783..776faae 100755 --- a/btcpay-setup.sh +++ b/btcpay-setup.sh @@ -52,7 +52,7 @@ Environment variables: LETSENCRYPT_EMAIL: A mail will be sent to this address if certificate expires and fail to renew automatically (eg. me@example.com) NBITCOIN_NETWORK: The type of network to use (eg. mainnet, testnet or regtest. Default: mainnet) LIGHTNING_ALIAS: An alias for your lightning network node if used - BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btx, btg, grs, ftc, via, doge, mona, none. Default: btc) + BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btx, btg, grs, ftc, via, doge, mona, dash, none. Default: btc) BTCPAYGEN_CRYPTO2: Second supported crypto currency (Default: empty) BTCPAYGEN_CRYPTON: N th supported crypto currency where N is maximum at maximum 9. (Default: none) BTCPAYGEN_REVERSEPROXY: Whether to use or not a reverse proxy. NGinx setup HTTPS for you. (eg. nginx, traefik, none. Default: nginx)