From e6187fc65733e50889e959187d8e2feaf1694604 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 12 Feb 2020 10:40:16 +0100 Subject: [PATCH 01/33] Replaces missing occurrences of pisa_cli for wt_cli --- apps/cli/DEPENDENCIES.md | 8 ++++---- apps/cli/INSTALL.md | 8 ++++---- apps/cli/help.py | 8 ++++---- apps/cli/sample_conf.py | 6 +++--- apps/cli/wt_cli.py | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/apps/cli/DEPENDENCIES.md b/apps/cli/DEPENDENCIES.md index abc6941..d5a4695 100644 --- a/apps/cli/DEPENDENCIES.md +++ b/apps/cli/DEPENDENCIES.md @@ -1,10 +1,10 @@ # Dependencies -`pisa-cli` has both system-wide and Python dependencies. This document walks you trough how to satisfy them. +`wt_cli` has both system-wide and Python dependencies. This document walks you trough how to satisfy them. ## System-wide dependencies -`pisa-cli` has the following system-wide dependencies: +`wt_cli` has the following system-wide dependencies: - `python3` - `pip3` @@ -27,7 +27,7 @@ It is also likely that, if `python3` is installed in our system, the `python` al python3 --version -If `python3` is installed but the `python` alias is not set to it, we should either set it, or use `python3` to run `pisa-cli`. +If `python3` is installed but the `python` alias is not set to it, we should either set it, or use `python3` to run `wt_cli`. Regarding `pip`, we can check what version is installed in our system (if any) by running: @@ -74,7 +74,7 @@ and for `pip3`: ## Python dependencies -`pisa-cli` has the following dependencies (which can be satisfied by using `pip install -r requirements.txt`): +`wt_cli` has the following dependencies (which can be satisfied by using `pip install -r requirements.txt`): - `cryptography` - `requests` \ No newline at end of file diff --git a/apps/cli/INSTALL.md b/apps/cli/INSTALL.md index 007fcf2..8127828 100644 --- a/apps/cli/INSTALL.md +++ b/apps/cli/INSTALL.md @@ -1,8 +1,8 @@ # Install -`pisa-cli` has some dependencies that can be satisfied by following [DEPENDENCIES.md](DEPENDENCIES.md). If your system already satisfies the dependencies, you can skip that part. +`wt_cli` has some dependencies that can be satisfied by following [DEPENDENCIES.md](DEPENDENCIES.md). If your system already satisfies the dependencies, you can skip that part. -In order to run `pisa-cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `apps` folder. You can do so by running: +In order to run `wt_cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `apps` folder. You can do so by running: export PYTHONPATH=$PYTHONPATH: @@ -14,7 +14,7 @@ You should also include the command in your `.bash_rc` to avoid having to run it echo 'export PYTHONPATH=$PYTHONPATH:' >> ~/.bash_rc -Once the `PYTHONPATH` is set, you should be able to run `pisa-cli` straightaway. Try it by running: +Once the `PYTHONPATH` is set, you should be able to run `wt_cli` straightaway. Try it by running: cd /apps/cli - python pisa-cli.py -h \ No newline at end of file + python wt_cli.py -h \ No newline at end of file diff --git a/apps/cli/help.py b/apps/cli/help.py index 298da7a..030c327 100644 --- a/apps/cli/help.py +++ b/apps/cli/help.py @@ -1,9 +1,9 @@ def help_add_appointment(): return ( "NAME:" - "\tpython pisa-cli add_appointment - Registers a json formatted appointment to the PISA server." + "\tpython wt_cli add_appointment - Registers a json formatted appointment to the PISA server." "\n\nUSAGE:" - "\tpython pisa-cli add_appointment [command options] appointment/path_to_appointment_file" + "\tpython wt_cli add_appointment [command options] appointment/path_to_appointment_file" "\n\nDESCRIPTION:" "\n\n\tRegisters a json formatted appointment to the PISA server." "\n\tif -f, --file *is* specified, then the command expects a path to a json file instead of a json encoded " @@ -17,9 +17,9 @@ def help_add_appointment(): def help_get_appointment(): return ( "NAME:" - "\tpython pisa-cli get_appointment - Gets json formatted data about an appointment from the PISA server." + "\tpython wt_cli get_appointment - Gets json formatted data about an appointment from the PISA server." "\n\nUSAGE:" - "\tpython pisa-cli get_appointment appointment_locator" + "\tpython wt_cli get_appointment appointment_locator" "\n\nDESCRIPTION:" "\n\n\tGets json formatted data about an appointment from the PISA server.\n" ) diff --git a/apps/cli/sample_conf.py b/apps/cli/sample_conf.py index ebe03ed..2ed6085 100644 --- a/apps/cli/sample_conf.py +++ b/apps/cli/sample_conf.py @@ -1,9 +1,9 @@ -# PISA-SERVER +# PISA-WT-SERVER DEFAULT_PISA_API_SERVER = "btc.pisa.watch" DEFAULT_PISA_API_PORT = 9814 -# PISA-CLI -DATA_FOLDER = "~/.pisa_btc/" +# WT-CLI +DATA_FOLDER = "~/.wt_cli/" CLIENT_LOG_FILE = "cli.log" APPOINTMENTS_FOLDER_NAME = "appointment_receipts" diff --git a/apps/cli/wt_cli.py b/apps/cli/wt_cli.py index c9b41b9..5f7f2d5 100644 --- a/apps/cli/wt_cli.py +++ b/apps/cli/wt_cli.py @@ -356,7 +356,7 @@ def get_appointment(locator): def show_usage(): return ( "USAGE: " - "\n\tpython pisa-cli.py [global options] command [command options] [arguments]" + "\n\tpython wt_cli.py [global options] command [command options] [arguments]" "\n\nCOMMANDS:" "\n\tadd_appointment \tRegisters a json formatted appointment to the PISA server." "\n\tget_appointment \tGets json formatted data about an appointment from the PISA server." @@ -365,7 +365,7 @@ def show_usage(): "\n\t-s, --server \tAPI server where to send the requests. Defaults to btc.pisa.watch (modifiable in " "__init__.py)" "\n\t-p, --port \tAPI port where to send the requests. Defaults to 9814 (modifiable in __init__.py)" - "\n\t-d, --debug \tshows debug information and stores it in pisa_cli.log" + "\n\t-d, --debug \tshows debug information and stores it in wt_cli.log" "\n\t-h --help \tshows this message." ) From a655eb8b984bc2e2196ae3aa468da2e943d437d1 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 13:35:02 +0100 Subject: [PATCH 02/33] Adds module info to INSTALL docs --- apps/cli/INSTALL.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/cli/INSTALL.md b/apps/cli/INSTALL.md index 8127828..556df77 100644 --- a/apps/cli/INSTALL.md +++ b/apps/cli/INSTALL.md @@ -2,6 +2,9 @@ `wt_cli` has some dependencies that can be satisfied by following [DEPENDENCIES.md](DEPENDENCIES.md). If your system already satisfies the dependencies, you can skip that part. +There are two ways of running `wt_cli`: adding the library to the `PYTHONPATH` env variable, or running it as a module + +## Modifying `PYTHONPATH` In order to run `wt_cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `apps` folder. You can do so by running: export PYTHONPATH=$PYTHONPATH: @@ -17,4 +20,11 @@ You should also include the command in your `.bash_rc` to avoid having to run it Once the `PYTHONPATH` is set, you should be able to run `wt_cli` straightaway. Try it by running: cd /apps/cli - python wt_cli.py -h \ No newline at end of file + python wt_cli.py -h + +## Running `wt_cli` as a module +Python code can be also run as a module, to do you need to use `python -m`. From `apps` **parent** directory run: + + python -m apps.cli.wt_cli -h + +Notice that if you run `wt_cli` as a module, you'll need to replace all the calls to `python wt_cli.py ` to `python -m apps.cli.wt_cli ` \ No newline at end of file From 6e40938733eb3618347c83409225441c55e51ab6 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 13:35:59 +0100 Subject: [PATCH 03/33] Updates PISA server endpoint --- apps/cli/sample_conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli/sample_conf.py b/apps/cli/sample_conf.py index 2ed6085..531c713 100644 --- a/apps/cli/sample_conf.py +++ b/apps/cli/sample_conf.py @@ -1,5 +1,5 @@ # PISA-WT-SERVER -DEFAULT_PISA_API_SERVER = "btc.pisa.watch" +DEFAULT_PISA_API_SERVER = "teos.pisa.watch" DEFAULT_PISA_API_PORT = 9814 # WT-CLI From 02e159faac9c20723294de0faf7e666033d81189 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 16:06:48 +0100 Subject: [PATCH 04/33] Moves EncryptedBlob to common --- apps/cli/wt_cli.py | 2 +- common/appointment.py | 4 ++-- {apps/cli => common}/blob.py | 0 common/cryptographer.py | 8 ++++---- {pisa => common}/encrypted_blob.py | 0 pisa/watcher.py | 6 +++--- test/apps/cli/unit/test_wt_cli.py | 2 +- test/common/unit/test_appointment.py | 2 +- test/{pisa => common}/unit/test_blob.py | 2 +- test/common/unit/test_cryptographer.py | 4 ++-- test/{pisa => common}/unit/test_encrypted_blob.py | 2 +- test/pisa/e2e/test_basic_e2e.py | 4 +--- test/pisa/unit/conftest.py | 2 +- 13 files changed, 18 insertions(+), 20 deletions(-) rename {apps/cli => common}/blob.py (100%) rename {pisa => common}/encrypted_blob.py (100%) rename test/{pisa => common}/unit/test_blob.py (92%) rename test/{pisa => common}/unit/test_encrypted_blob.py (89%) diff --git a/apps/cli/wt_cli.py b/apps/cli/wt_cli.py index 5f7f2d5..0ecc013 100644 --- a/apps/cli/wt_cli.py +++ b/apps/cli/wt_cli.py @@ -11,7 +11,7 @@ from uuid import uuid4 from apps.cli import config, LOG_PREFIX from apps.cli.help import help_add_appointment, help_get_appointment -from apps.cli.blob import Blob +from common.blob import Blob import common.cryptographer from common import constants diff --git a/common/appointment.py b/common/appointment.py index 7b21471..e376668 100644 --- a/common/appointment.py +++ b/common/appointment.py @@ -2,7 +2,7 @@ import json import struct from binascii import unhexlify -from pisa.encrypted_blob import EncryptedBlob +from common.encrypted_blob import EncryptedBlob class Appointment: @@ -16,7 +16,7 @@ class Appointment: end_time (:mod:`int`): The block height where the tower will stop watching for breaches. to_self_delay (:mod:`int`): The ``to_self_delay`` encoded in the ``csv`` of the ``htlc`` that this appointment is covering. - encrypted_blob (:obj:`EncryptedBlob `): An ``EncryptedBlob`` object + encrypted_blob (:obj:`EncryptedBlob `): An ``EncryptedBlob`` object containing an encrypted penalty transaction. The tower will decrypt it and broadcast the penalty transaction upon seeing a breach on the blockchain. """ diff --git a/apps/cli/blob.py b/common/blob.py similarity index 100% rename from apps/cli/blob.py rename to common/blob.py diff --git a/common/cryptographer.py b/common/cryptographer.py index 6519620..8cac45f 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -49,12 +49,12 @@ class Cryptographer: @staticmethod def encrypt(blob, secret, rtype="str"): """ - Encrypts a given :mod:`Blob ` data using ``CHACHA20POLY1305``. + Encrypts a given :mod:`Blob ` data using ``CHACHA20POLY1305``. ``SHA256(secret)`` is used as ``key``, and ``0 (12-byte)`` as ``iv``. Args: - blob (:mod:`Blob `): a ``Blob`` object containing a raw penalty transaction. + blob (:mod:`Blob `): a ``Blob`` object containing a raw penalty transaction. secret (:mod:`str`): a value to used to derive the encryption key. Should be the dispute txid. rtype(:mod:`str`): the return type for the encrypted value. Can be either ``'str'`` or ``'bytes'``. @@ -93,12 +93,12 @@ class Cryptographer: # ToDo: #20-test-tx-decrypting-edge-cases def decrypt(encrypted_blob, secret, rtype="str"): """ - Decrypts a given :mod:`EncryptedBlob ` using ``CHACHA20POLY1305``. + Decrypts a given :mod:`EncryptedBlob ` using ``CHACHA20POLY1305``. ``SHA256(secret)`` is used as ``key``, and ``0 (12-byte)`` as ``iv``. Args: - encrypted_blob(:mod:`EncryptedBlob `): an ``EncryptedBlob`` potentially + encrypted_blob(:mod:`EncryptedBlob `): an ``EncryptedBlob`` potentially containing a penalty transaction. secret (:mod:`str`): a value to used to derive the decryption key. Should be the dispute txid. rtype(:mod:`str`): the return type for the decrypted value. Can be either ``'str'`` or ``'bytes'``. diff --git a/pisa/encrypted_blob.py b/common/encrypted_blob.py similarity index 100% rename from pisa/encrypted_blob.py rename to common/encrypted_blob.py diff --git a/pisa/watcher.py b/pisa/watcher.py index 281de92..c06bee3 100644 --- a/pisa/watcher.py +++ b/pisa/watcher.py @@ -24,7 +24,7 @@ class Watcher: The :class:`Watcher` keeps track of the accepted appointments in ``appointments`` and, for new received block, checks if any breach has happened by comparing the txids with the appointment locators. If a breach is seen, the - :obj:`EncryptedBlob ` of the corresponding appointment is decrypted and the data + :obj:`EncryptedBlob ` of the corresponding appointment is decrypted and the data is passed to the :obj:`Responder `. If an appointment reaches its end with no breach, the data is simply deleted. @@ -81,7 +81,7 @@ class Watcher: the blockchain (``do_watch``) until ``appointments`` is empty. Once a breach is seen on the blockchain, the :obj:`Watcher` will decrypt the corresponding - :obj:`EncryptedBlob ` and pass the information to the + :obj:`EncryptedBlob ` and pass the information to the :obj:`Responder `. The tower may store multiple appointments with the same ``locator`` to avoid DoS attacks based on data @@ -232,7 +232,7 @@ class Watcher: """ Filters what of the found breaches contain valid transaction data. - The :obj:`Watcher` cannot if a given :obj:`EncryptedBlob ` contains a valid + The :obj:`Watcher` cannot if a given :obj:`EncryptedBlob ` contains a valid transaction until a breach if seen. Blobs that contain arbitrary data are dropped and not sent to the :obj:`Responder `. diff --git a/test/apps/cli/unit/test_wt_cli.py b/test/apps/cli/unit/test_wt_cli.py index b05164e..3545eef 100644 --- a/test/apps/cli/unit/test_wt_cli.py +++ b/test/apps/cli/unit/test_wt_cli.py @@ -14,7 +14,7 @@ from common.tools import compute_locator from common.appointment import Appointment from common.cryptographer import Cryptographer -from apps.cli.blob import Blob +from common.blob import Blob import apps.cli.wt_cli as wt_cli from test.apps.cli.unit.conftest import get_random_value_hex diff --git a/test/common/unit/test_appointment.py b/test/common/unit/test_appointment.py index 8087138..f83538f 100644 --- a/test/common/unit/test_appointment.py +++ b/test/common/unit/test_appointment.py @@ -4,7 +4,7 @@ import binascii from pytest import fixture from common.appointment import Appointment -from pisa.encrypted_blob import EncryptedBlob +from common.encrypted_blob import EncryptedBlob from test.pisa.unit.conftest import get_random_value_hex diff --git a/test/pisa/unit/test_blob.py b/test/common/unit/test_blob.py similarity index 92% rename from test/pisa/unit/test_blob.py rename to test/common/unit/test_blob.py index a12de8b..7172330 100644 --- a/test/pisa/unit/test_blob.py +++ b/test/common/unit/test_blob.py @@ -1,6 +1,6 @@ from binascii import unhexlify -from apps.cli.blob import Blob +from common.blob import Blob from test.pisa.unit.conftest import get_random_value_hex diff --git a/test/common/unit/test_cryptographer.py b/test/common/unit/test_cryptographer.py index 728e1fd..02b2727 100644 --- a/test/common/unit/test_cryptographer.py +++ b/test/common/unit/test_cryptographer.py @@ -5,10 +5,10 @@ from cryptography.hazmat.primitives.asymmetric import ec from cryptography.hazmat.primitives import serialization import common.cryptographer -from apps.cli.blob import Blob +from common.blob import Blob from common.logger import Logger from common.cryptographer import Cryptographer -from pisa.encrypted_blob import EncryptedBlob +from common.encrypted_blob import EncryptedBlob from test.common.unit.conftest import get_random_value_hex common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="") diff --git a/test/pisa/unit/test_encrypted_blob.py b/test/common/unit/test_encrypted_blob.py similarity index 89% rename from test/pisa/unit/test_encrypted_blob.py rename to test/common/unit/test_encrypted_blob.py index 64add70..b977dbc 100644 --- a/test/pisa/unit/test_encrypted_blob.py +++ b/test/common/unit/test_encrypted_blob.py @@ -1,4 +1,4 @@ -from pisa.encrypted_blob import EncryptedBlob +from common.encrypted_blob import EncryptedBlob from test.pisa.unit.conftest import get_random_value_hex diff --git a/test/pisa/e2e/test_basic_e2e.py b/test/pisa/e2e/test_basic_e2e.py index 3124e7c..8fea9c7 100644 --- a/test/pisa/e2e/test_basic_e2e.py +++ b/test/pisa/e2e/test_basic_e2e.py @@ -1,13 +1,11 @@ import json -import binascii from time import sleep from riemann.tx import Tx from pisa import config from pisa import HOST, PORT from apps.cli import wt_cli -from apps.cli.blob import Blob -from apps.cli import config as cli_conf +from common.blob import Blob import common.cryptographer from common.logger import Logger diff --git a/test/pisa/unit/conftest.py b/test/pisa/unit/conftest.py index 6766faa..28d282d 100644 --- a/test/pisa/unit/conftest.py +++ b/test/pisa/unit/conftest.py @@ -11,7 +11,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec from cryptography.hazmat.primitives import serialization -from apps.cli.blob import Blob +from common.blob import Blob from pisa.responder import TransactionTracker from pisa.tools import bitcoin_cli from pisa.db_manager import DBManager From d8977acd2d1258a56ad713edefbffe79d5a9ede2 Mon Sep 17 00:00:00 2001 From: Chris Buckland Date: Thu, 13 Feb 2020 15:18:51 +0100 Subject: [PATCH 05/33] Typo --- apps/cli/DEPENDENCIES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cli/DEPENDENCIES.md b/apps/cli/DEPENDENCIES.md index d5a4695..ef2e951 100644 --- a/apps/cli/DEPENDENCIES.md +++ b/apps/cli/DEPENDENCIES.md @@ -1,6 +1,6 @@ # Dependencies -`wt_cli` has both system-wide and Python dependencies. This document walks you trough how to satisfy them. +`wt_cli` has both system-wide and Python dependencies. This document walks you through how to satisfy them. ## System-wide dependencies @@ -77,4 +77,4 @@ and for `pip3`: `wt_cli` has the following dependencies (which can be satisfied by using `pip install -r requirements.txt`): - `cryptography` -- `requests` \ No newline at end of file +- `requests` From e2e27d3e9cdce6c276c21451e18057f6f450fa5f Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 16:23:00 +0100 Subject: [PATCH 06/33] Adds config file section to install --- apps/cli/INSTALL.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/cli/INSTALL.md b/apps/cli/INSTALL.md index 556df77..ccf7f90 100644 --- a/apps/cli/INSTALL.md +++ b/apps/cli/INSTALL.md @@ -27,4 +27,11 @@ Python code can be also run as a module, to do you need to use `python -m`. From python -m apps.cli.wt_cli -h -Notice that if you run `wt_cli` as a module, you'll need to replace all the calls to `python wt_cli.py ` to `python -m apps.cli.wt_cli ` \ No newline at end of file +Notice that if you run `wt_cli` as a module, you'll need to replace all the calls to `python wt_cli.py ` to `python -m apps.cli.wt_cli ` + +## Create a config file +You need to create a config file with the basic configuration parameters for `wt_cli`. To do so run: + + cp /apps/cli/sample_conf.py /apps/cli/conf.py + +You can change any of the config options in `conf.py` after that, like the path to your user data directory. \ No newline at end of file From adf883868e97cdbe6e56ab13dd8cc30c6d882b09 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 16:42:29 +0100 Subject: [PATCH 07/33] Improves config section --- apps/cli/INSTALL.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/cli/INSTALL.md b/apps/cli/INSTALL.md index ccf7f90..80068e0 100644 --- a/apps/cli/INSTALL.md +++ b/apps/cli/INSTALL.md @@ -29,9 +29,7 @@ Python code can be also run as a module, to do you need to use `python -m`. From Notice that if you run `wt_cli` as a module, you'll need to replace all the calls to `python wt_cli.py ` to `python -m apps.cli.wt_cli ` -## Create a config file -You need to create a config file with the basic configuration parameters for `wt_cli`. To do so run: +## Modify configuration parameters +If you'd like to modify some of the configuration defaults (such as the user directory, where the logs and appointment receipts will be stored) you can do so in the config file located at: - cp /apps/cli/sample_conf.py /apps/cli/conf.py - -You can change any of the config options in `conf.py` after that, like the path to your user data directory. \ No newline at end of file + /apps/cli/conf.py \ No newline at end of file From 7e298244527bcaaf175fb016ea199bc65b8da927 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 16:51:44 +0100 Subject: [PATCH 08/33] Improves API log and adds logs for get_appointment --- pisa/api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pisa/api.py b/pisa/api.py index e31d0d2..bf6665c 100644 --- a/pisa/api.py +++ b/pisa/api.py @@ -39,7 +39,7 @@ class API: remote_addr = request.environ.get("REMOTE_ADDR") remote_port = request.environ.get("REMOTE_PORT") - logger.info("Connection accepted", from_addr_port="{}:{}".format(remote_addr, remote_port)) + logger.info("Received add_appointment request", from_addr_port="{}:{}".format(remote_addr, remote_port)) # Check content type once if properly defined request_data = json.loads(request.get_json()) @@ -101,10 +101,16 @@ class API: - Appointments hold by the :obj:`Responder ` are flagged as ``dispute_triggered``. - Unknown appointments are flagged as ``not_found``. """ + remote_addr = request.environ.get("REMOTE_ADDR") + remote_port = request.environ.get("REMOTE_PORT") locator = request.args.get("locator") response = [] + logger.info( + "Received get_appointment request", from_addr_port="{}:{}".format(remote_addr, remote_port), locator=locator + ) + # ToDo: #15-add-system-monitor if not isinstance(locator, str) or len(locator) != LOCATOR_LEN_HEX: response.append({"locator": locator, "status": "not_found"}) From 3b5ff2e92c0b07c77d10ff873288f6c3b6b39220 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 16:59:06 +0100 Subject: [PATCH 09/33] Removes get_block_count endpoint get_block_count was only for dummy testing and it's not used anymore --- pisa/api.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/pisa/api.py b/pisa/api.py index bf6665c..a387b40 100644 --- a/pisa/api.py +++ b/pisa/api.py @@ -7,7 +7,6 @@ from pisa import HOST, PORT, LOG_PREFIX from common.logger import Logger from pisa.inspector import Inspector from common.appointment import Appointment -from pisa.block_processor import BlockProcessor from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, LOCATOR_LEN_HEX @@ -83,7 +82,7 @@ class API: else: return jsonify({"error": error}), rcode - # FIXME: THE NEXT THREE API ENDPOINTS ARE FOR TESTING AND SHOULD BE REMOVED / PROPERLY MANAGED BEFORE PRODUCTION! + # FIXME: THE NEXT TWO API ENDPOINTS ARE FOR TESTING AND SHOULD BE REMOVED / PROPERLY MANAGED BEFORE PRODUCTION! # ToDo: #17-add-api-keys def get_appointment(self): """ @@ -168,21 +167,6 @@ class API: return response - @staticmethod - def get_block_count(): - """ - Provides the block height of the Watchtower. - - This is a testing endpoint that (most likely) will be removed in production. Its purpose is to give information - to testers about the current block so they can define a dummy appointment without having to run a bitcoin node. - - Returns: - :obj:`dict`: A json encoded dictionary containing the block height. - - """ - - return jsonify({"block_count": BlockProcessor.get_block_count()}) - def start(self): """ This function starts the Flask server used to run the API. Adds all the routes to the functions listed above. @@ -192,7 +176,6 @@ class API: "/": (self.add_appointment, ["POST"]), "/get_appointment": (self.get_appointment, ["GET"]), "/get_all_appointments": (self.get_all_appointments, ["GET"]), - "/get_block_count": (self.get_block_count, ["GET"]), } for url, params in routes.items(): From f00e4562e65f11d736c5be8f5fab3713cf3cdfa9 Mon Sep 17 00:00:00 2001 From: Chris Buckland Date: Thu, 13 Feb 2020 17:30:04 +0100 Subject: [PATCH 10/33] Reformatted json appointment block --- apps/cli/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/cli/README.md b/apps/cli/README.md index ddea17a..93fbca3 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -36,8 +36,7 @@ This command is used to register appointments to the PISA server. Appointments * "tx_id": tx_id, "start_time": s, "end_time": e, - "dispute_delta": d - } + "dispute_delta": d } `tx` **must** be the raw penalty transaction that will be encrypted before sent to the PISA server. `type(tx) = hex encoded str` From f24bb75e07d499cf259cabba3923f8be4a4337cb Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 17:45:43 +0100 Subject: [PATCH 11/33] Fixes formatting issues in cli/README --- apps/cli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli/README.md b/apps/cli/README.md index 93fbca3..1b96892 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -117,7 +117,7 @@ or 1. Generate a new dummy appointment. **Note:** this appointment will never be fulfilled (it will eventually expire) since it does not corresopond to a valid transaction. However it can be used to interact with the PISA API. ``` -echo '{"tx": "4615a58815475ab8145b6bb90b1268a0dbb02e344ddd483f45052bec1f15b1951c1ee7f070a0993da395a5ee92ea3a1c184b5ffdb2507164bf1f8c1364155d48bdbc882eee0868ca69864a807f213f538990ad16f56d7dfb28a18e69e3f31ae9adad229e3244073b7d643b4597ec88bf247b9f73f301b0f25ae8207b02b7709c271da98af19f1db276ac48ba64f099644af1ae2c90edb7def5e8589a1bb17cc72ac42ecf07dd29cff91823938fd0d772c2c92b7ab050f8837efd46197c9b2b3f", "tx_id": "0b9510d92a50c1d67c6f7fc5d47908d96b3eccdea093d89bcbaf05bcfebdd951", "start_time": 0, "end_time": 0, "to_self_delay": 20}' > dummy_appointment_data.json + echo '{"tx": "4615a58815475ab8145b6bb90b1268a0dbb02e344ddd483f45052bec1f15b1951c1ee7f070a0993da395a5ee92ea3a1c184b5ffdb2507164bf1f8c1364155d48bdbc882eee0868ca69864a807f213f538990ad16f56d7dfb28a18e69e3f31ae9adad229e3244073b7d643b4597ec88bf247b9f73f301b0f25ae8207b02b7709c271da98af19f1db276ac48ba64f099644af1ae2c90edb7def5e8589a1bb17cc72ac42ecf07dd29cff91823938fd0d772c2c92b7ab050f8837efd46197c9b2b3f", "tx_id": "0b9510d92a50c1d67c6f7fc5d47908d96b3eccdea093d89bcbaf05bcfebdd951", "start_time": 0, "end_time": 0, "to_self_delay": 20}' > dummy_appointment_data.json ``` That will create a json file that follows the appointment data structure filled with dummy data and store it in `dummy_appointment_data.json`. **Note**: You'll need to update the `start_time` and `end_time` to match valid block heights. From 3ba5db533d747e7d6d53c709dc8e10302866dafc Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 18:37:02 +0100 Subject: [PATCH 12/33] Fixes json parsing of None when no json was postesd to the API h/t @yahgwai for finding this. --- apps/cli/README.md | 2 +- pisa/api.py | 48 ++++++++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/apps/cli/README.md b/apps/cli/README.md index 1b96892..6884d3b 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -36,7 +36,7 @@ This command is used to register appointments to the PISA server. Appointments * "tx_id": tx_id, "start_time": s, "end_time": e, - "dispute_delta": d } + "to_self_delay": d } `tx` **must** be the raw penalty transaction that will be encrypted before sent to the PISA server. `type(tx) = hex encoded str` diff --git a/pisa/api.py b/pisa/api.py index a387b40..efb519a 100644 --- a/pisa/api.py +++ b/pisa/api.py @@ -40,35 +40,41 @@ class API: logger.info("Received add_appointment request", from_addr_port="{}:{}".format(remote_addr, remote_port)) - # Check content type once if properly defined - request_data = json.loads(request.get_json()) - inspector = Inspector(self.config) - appointment = inspector.inspect( - request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key") - ) + if request.is_json: + # Check content type once if properly defined + request_data = json.loads(request.get_json()) + inspector = Inspector(self.config) + appointment = inspector.inspect( + request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key") + ) - error = None - response = None + error = None + response = None - if type(appointment) == Appointment: - appointment_added, signature = self.watcher.add_appointment(appointment) + if type(appointment) == Appointment: + appointment_added, signature = self.watcher.add_appointment(appointment) - if appointment_added: - rcode = HTTP_OK - response = {"locator": appointment.locator, "signature": signature} + if appointment_added: + rcode = HTTP_OK + response = {"locator": appointment.locator, "signature": signature} + + else: + rcode = HTTP_SERVICE_UNAVAILABLE + error = "appointment rejected" + + elif type(appointment) == tuple: + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Error {}: {}".format(appointment[0], appointment[1]) else: - rcode = HTTP_SERVICE_UNAVAILABLE - error = "appointment rejected" - - elif type(appointment) == tuple: - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Error {}: {}".format(appointment[0], appointment[1]) + # We should never end up here, since inspect only returns appointments or tuples. Just in case. + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Request does not match the standard" else: - # We should never end up here, since inspect only returns appointments or tuples. Just in case. rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Request does not match the standard" + error = "appointment rejected. Request is not json encoded" + response = None logger.info( "Sending response and disconnecting", From 43975df8942490a1d0e64011b36fc0d1f1145636 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 19:00:15 +0100 Subject: [PATCH 13/33] Changes cli to be protocol agnostic and defines https on config file --- apps/cli/sample_conf.py | 4 ++-- apps/cli/wt_cli.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/cli/sample_conf.py b/apps/cli/sample_conf.py index 531c713..67396fd 100644 --- a/apps/cli/sample_conf.py +++ b/apps/cli/sample_conf.py @@ -1,6 +1,6 @@ # PISA-WT-SERVER -DEFAULT_PISA_API_SERVER = "teos.pisa.watch" -DEFAULT_PISA_API_PORT = 9814 +DEFAULT_PISA_API_SERVER = "https://teos.pisa.watch" +DEFAULT_PISA_API_PORT = 443 # WT-CLI DATA_FOLDER = "~/.wt_cli/" diff --git a/apps/cli/wt_cli.py b/apps/cli/wt_cli.py index 0ecc013..1da8936 100644 --- a/apps/cli/wt_cli.py +++ b/apps/cli/wt_cli.py @@ -235,7 +235,7 @@ def post_appointment(data): logger.info("Sending appointment to PISA") try: - add_appointment_endpoint = "http://{}:{}".format(pisa_api_server, pisa_api_port) + add_appointment_endpoint = "{}:{}".format(pisa_api_server, pisa_api_port) return requests.post(url=add_appointment_endpoint, json=json.dumps(data), timeout=5) except ConnectTimeout: @@ -337,7 +337,7 @@ def get_appointment(locator): logger.error("The provided locator is not valid", locator=locator) return None - get_appointment_endpoint = "http://{}:{}/get_appointment".format(pisa_api_server, pisa_api_port) + get_appointment_endpoint = "{}:{}/get_appointment".format(pisa_api_server, pisa_api_port) parameters = "?locator={}".format(locator) try: From 452dbb6bee9dbf4bb703855468044a3c70c25911 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 19:02:30 +0100 Subject: [PATCH 14/33] Updates check_appointment to get_appointment and justice to penalty --- apps/cli/PISA-API.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/cli/PISA-API.md b/apps/cli/PISA-API.md index 60038b1..bf52edf 100644 --- a/apps/cli/PISA-API.md +++ b/apps/cli/PISA-API.md @@ -2,7 +2,7 @@ ### Disclaimer: Everything in here is experimental and subject to change. -The PISA REST API consists, currently, of two endpoints: `/` and `/check_appointment` +The PISA REST API consists, currently, of two endpoints: `/` and `/get_appointment` `/` is the default endpoint, and is where the appointments should be sent to. `/` accepts `HTTP POST` requests only, with json request body, where data must match the following format: @@ -36,21 +36,21 @@ The API will return a `application/json` HTTP response code `200/OK` if the appo "to_self_delay": 20, "encrypted_blob": "6c7687a97e874363e1c2b9a08386125e09ea000a9b4330feb33a5c698265f3565c267554e6fdd7b0544ced026aaab73c255bcc97c18eb9fa704d9cc5f1c83adaf921de7ba62b2b6ddb1bda7775288019ec3708642e738eddc22882abf5b3f4e34ef2d4077ed23e135f7fe22caaec845982918e7df4a3f949cadd2d3e7c541b1dbf77daf64e7ed61531aaa487b468581b5aa7b1da81e2617e351c9d5cf445e3391c3fea4497aaa7ad286552759791b9caa5e4c055d1b38adfceddb1ef2b99e3b467dd0b0b13ce863c1bf6b6f24543c30d"} -# Check appointment +# Get appointment -`/check_appointment` is an endpoint provided to check the status of the appointments sent to PISA. The endpoint is accessible without any type of authentication for now. `/check_appointment` accepts `HTTP GET` requests only, where the data to be provided must be the locator of an appointment. The query must match the following format: +`/get_appointment` is an endpoint provided to check the status of the appointments sent to PISA. The endpoint is accessible without any type of authentication for now. `/get_appointment` accepts `HTTP GET` requests only, where the data to be provided must be the locator of an appointment. The query must match the following format: -`http://pisa_server:pisa_port/check_appointment?locator=appointment_locator` +`https://pisa_server:pisa_port/get_appointment?locator=appointment_locator` ### Appointment can be in three states - `not_found`: meaning the locator is not recognised by the API. This could either mean the locator is wrong, or the appointment has already been fulfilled. -- `being_watched`: the appointment has been accepted by the PISA server and it's being watched at the moment. This stage means that the dispute transaction has not been seen yet, and therefore no justice transaction has been published. -- `dispute_responded`: the dispute was found by the watcher and the corresponding justice transaction has been broadcast by the node. In this stage PISA is actively monitoring until the justice transaction reaches enough confirmations and making sure no fork occurs in the meantime. +- `being_watched`: the appointment has been accepted by the PISA server and it's being watched at the moment. This stage means that the dispute transaction has not been seen yet, and therefore no penalty transaction has been published. +- `dispute_responded`: the dispute was found by the watcher and the corresponding penalty transaction has been broadcast by the node. In this stage PISA is actively monitoring until the penalty transaction reaches enough confirmations and making sure no fork occurs in the meantime. -### Check appointment response formats +### Get appointment response formats -`/check_appointment` will always reply with `json` containing the information about the requested appointment. The structure is as follows: +`/get_appointment` will always reply with `json` containing the information about the requested appointment. The structure is as follows: **not_found** From 1471e8985549ee798681b3ae121f59baf0e5e2ac Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 19:11:51 +0100 Subject: [PATCH 15/33] Adds exception handling for no schema and timeout --- apps/cli/wt_cli.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/cli/wt_cli.py b/apps/cli/wt_cli.py index 1da8936..6a08b3b 100644 --- a/apps/cli/wt_cli.py +++ b/apps/cli/wt_cli.py @@ -246,6 +246,12 @@ def post_appointment(data): logger.error("Can't connect to PISA API. Server cannot be reached") return None + except requests.exceptions.InvalidSchema: + logger.error("No transport protocol found. Have you missed http(s):// in the server url?") + + except requests.exceptions.Timeout: + logger.error("The request timed out") + def process_post_appointment_response(response): """ @@ -352,6 +358,12 @@ def get_appointment(locator): logger.error("Can't connect to PISA API. Server cannot be reached") return None + except requests.exceptions.InvalidSchema: + logger.error("No transport protocol found. Have you missed http(s):// in the server url?") + + except requests.exceptions.Timeout: + logger.error("The request timed out") + def show_usage(): return ( From ead74199e55e308825b8e8e0c0f1e091817e9200 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 13 Feb 2020 19:14:13 +0100 Subject: [PATCH 16/33] Adds schema on cli and e2e tests --- test/apps/cli/unit/test_wt_cli.py | 4 ++-- test/pisa/e2e/test_basic_e2e.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/apps/cli/unit/test_wt_cli.py b/test/apps/cli/unit/test_wt_cli.py index 3545eef..5fe2699 100644 --- a/test/apps/cli/unit/test_wt_cli.py +++ b/test/apps/cli/unit/test_wt_cli.py @@ -38,9 +38,9 @@ dummy_pk_der = dummy_pk.public_bytes( # Replace the key in the module with a key we control for the tests wt_cli.pisa_public_key = dummy_pk # Replace endpoint with dummy one -wt_cli.pisa_api_server = "dummy.com" +wt_cli.pisa_api_server = "https://dummy.com" wt_cli.pisa_api_port = 12345 -pisa_endpoint = "http://{}:{}/".format(wt_cli.pisa_api_server, wt_cli.pisa_api_port) +pisa_endpoint = "{}:{}/".format(wt_cli.pisa_api_server, wt_cli.pisa_api_port) dummy_appointment_request = { "tx": get_random_value_hex(192), diff --git a/test/pisa/e2e/test_basic_e2e.py b/test/pisa/e2e/test_basic_e2e.py index 8fea9c7..96ceed8 100644 --- a/test/pisa/e2e/test_basic_e2e.py +++ b/test/pisa/e2e/test_basic_e2e.py @@ -25,7 +25,7 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="") # We'll use wt_cli to add appointments. The expected input format is a list of arguments with a json-encoded # appointment -wt_cli.pisa_api_server = HOST +wt_cli.pisa_api_server = "http://{}".format(HOST) wt_cli.pisa_api_port = PORT # Run pisad From de2036b19bb3a56d209b30b6898d795004a8d3f4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 12:42:47 +0100 Subject: [PATCH 17/33] Fixes cli console log levels so Cryptographer's sensitive info is only logged in disk --- common/cryptographer.py | 2 +- common/logger.py | 4 ++-- common/tools.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/cryptographer.py b/common/cryptographer.py index 8cac45f..c701f5f 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -78,7 +78,7 @@ class Cryptographer: sk = sha256(unhexlify(secret)).digest() nonce = bytearray(12) - logger.info("Encrypting blob", sk=hexlify(sk).decode(), nonce=hexlify(nonce).decode(), blob=blob.data) + logger.debug("Encrypting blob", sk=hexlify(sk).decode(), nonce=hexlify(nonce).decode(), blob=blob.data) # Encrypt the data cipher = ChaCha20Poly1305(sk) diff --git a/common/logger.py b/common/logger.py index b175ebf..136b330 100644 --- a/common/logger.py +++ b/common/logger.py @@ -60,7 +60,7 @@ class Logger: def debug(self, msg, **kwargs): """ - Logs an ``DEBUG`` level message to stdout and file. + Logs a ``DEBUG`` level message to stdout and file. Args: msg (:obj:`str`): the message to be logged. @@ -84,7 +84,7 @@ class Logger: def warning(self, msg, **kwargs): """ - Logs an ``WARNING`` level message to stdout and file. + Logs a ``WARNING`` level message to stdout and file. Args: msg (:obj:`str`): the message to be logged. diff --git a/common/tools.py b/common/tools.py index 0c131da..55dfe09 100644 --- a/common/tools.py +++ b/common/tools.py @@ -129,10 +129,10 @@ def setup_logging(log_file_path, log_name_prefix): # Create the file logger f_logger = logging.getLogger("{}_file_log".format(log_name_prefix)) - f_logger.setLevel(logging.INFO) + f_logger.setLevel(logging.DEBUG) fh = logging.FileHandler(log_file_path) - fh.setLevel(logging.INFO) + fh.setLevel(logging.DEBUG) fh_formatter = logging.Formatter("%(message)s") fh.setFormatter(fh_formatter) f_logger.addHandler(fh) From 4c9ddd082713157d43f39ae6994c3730880970e4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 13:13:06 +0100 Subject: [PATCH 18/33] Sets limits for star/end time and to_self_delay for the alpha --- pisa/inspector.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/pisa/inspector.py b/pisa/inspector.py index ee5bd10..a8f9b1a 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -19,6 +19,9 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_ # stored + blacklist if multiple wrong requests are received. +BLOCKS_IN_A_MONTH = 4320 # 4320 = roughly a month in blocks + + class Inspector: """ The :class:`Inspector` class is in charge of verifying that the appointment data provided by the user is correct. @@ -57,7 +60,10 @@ class Inspector: appointment_data.get("end_time"), appointment_data.get("start_time"), block_height ) if rcode == 0: - rcode, message = self.check_to_self_delay(appointment_data.get("to_self_delay")) + rcode, message = self.check_to_self_delay( + appointment_data.get("to_self_delay"), + appointment_data.get("end_time") - appointment_data.get("start_time"), + ) if rcode == 0: rcode, message = self.check_blob(appointment_data.get("encrypted_blob")) # if rcode == 0: @@ -161,7 +167,14 @@ class Inspector: if start_time < block_height: message = "start_time is in the past" else: - message = "start_time is too close to current height" + message = ( + "start_time is too close to current height. " + "Accepted times are: [current_height+1, current_height+2]" + ) + + elif start_time > block_height + 6: + rcode = errors.APPOINTMENT_FIELD_TOO_BIG + message = "start_time is too far in the future. Accepted start times are up to 6 blocks in the future" if message is not None: logger.error(message) @@ -206,6 +219,10 @@ class Inspector: rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE message = "wrong end_time data type ({})".format(t) + elif end_time > block_height + BLOCKS_IN_A_MONTH: # 4320 = roughly a month in blocks + rcode = errors.APPOINTMENT_FIELD_TOO_BIG + message = "end_time should be within the next month (>= current_height + 4320)" + elif start_time >= end_time: rcode = errors.APPOINTMENT_FIELD_TOO_SMALL if start_time > end_time: @@ -225,7 +242,7 @@ class Inspector: return rcode, message - def check_to_self_delay(self, to_self_delay): + def check_to_self_delay(self, to_self_delay, start_end_diff): """ Checks if the provided ``to_self_delay`` is correct. @@ -258,6 +275,12 @@ class Inspector: rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE message = "wrong to_self_delay data type ({})".format(t) + elif to_self_delay > start_end_diff: + rcode = errors.APPOINTMENT_FIELD_TOO_BIG + message = "to_self_delay can't be bigger than the appointment time {} ({})".format( + start_end_diff, to_self_delay + ) + elif to_self_delay < self.config.get("MIN_TO_SELF_DELAY"): rcode = errors.APPOINTMENT_FIELD_TOO_SMALL message = "to_self_delay too small. The to_self_delay should be at least {} (current: {})".format( From 99a9b2f33eb0345e64d2b37192db614a4e39c70a Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 13:31:23 +0100 Subject: [PATCH 19/33] Improve return messages from 83df910d5954bf1943e504b84e3aa5e0eb8b21d7 --- pisa/inspector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pisa/inspector.py b/pisa/inspector.py index a8f9b1a..f4c254e 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -221,7 +221,7 @@ class Inspector: elif end_time > block_height + BLOCKS_IN_A_MONTH: # 4320 = roughly a month in blocks rcode = errors.APPOINTMENT_FIELD_TOO_BIG - message = "end_time should be within the next month (>= current_height + 4320)" + message = "end_time should be within the next month (<= current_height + 4320)" elif start_time >= end_time: rcode = errors.APPOINTMENT_FIELD_TOO_SMALL @@ -277,8 +277,8 @@ class Inspector: elif to_self_delay > start_end_diff: rcode = errors.APPOINTMENT_FIELD_TOO_BIG - message = "to_self_delay can't be bigger than the appointment time {} ({})".format( - start_end_diff, to_self_delay + message = "to_self_delay can't be bigger than the appointment time ({} > {})".format( + to_self_delay, start_end_diff ) elif to_self_delay < self.config.get("MIN_TO_SELF_DELAY"): From ec8418294bd265bee06828ca3dbeea35fe94d3d4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 13:47:08 +0100 Subject: [PATCH 20/33] Changes to_self_delay restrictions to fit the nLockTime field --- pisa/inspector.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pisa/inspector.py b/pisa/inspector.py index f4c254e..2c135e7 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -60,10 +60,7 @@ class Inspector: appointment_data.get("end_time"), appointment_data.get("start_time"), block_height ) if rcode == 0: - rcode, message = self.check_to_self_delay( - appointment_data.get("to_self_delay"), - appointment_data.get("end_time") - appointment_data.get("start_time"), - ) + rcode, message = self.check_to_self_delay(appointment_data.get("to_self_delay")) if rcode == 0: rcode, message = self.check_blob(appointment_data.get("encrypted_blob")) # if rcode == 0: @@ -242,7 +239,7 @@ class Inspector: return rcode, message - def check_to_self_delay(self, to_self_delay, start_end_diff): + def check_to_self_delay(self, to_self_delay): """ Checks if the provided ``to_self_delay`` is correct. @@ -275,10 +272,10 @@ class Inspector: rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE message = "wrong to_self_delay data type ({})".format(t) - elif to_self_delay > start_end_diff: + elif to_self_delay > pow(2, 32): rcode = errors.APPOINTMENT_FIELD_TOO_BIG - message = "to_self_delay can't be bigger than the appointment time ({} > {})".format( - to_self_delay, start_end_diff + message = "to_self_delay must fit the transaction nLockTime field ({} > {})".format( + to_self_delay, pow(2, 32) ) elif to_self_delay < self.config.get("MIN_TO_SELF_DELAY"): From 0fd74bc6180421b7a1826370897f65cb8a8eb4a6 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 13:52:36 +0100 Subject: [PATCH 21/33] Updates README with alpha start/end time restrictions --- apps/cli/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/cli/README.md b/apps/cli/README.md index 6884d3b..3ce7472 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -50,6 +50,12 @@ This command is used to register appointments to the PISA server. Appointments * The API will return a `application/json` HTTP response code `200/OK` if the appointment is accepted, with the locator encoded in the response text, or a `400/Bad Request` if the appointment is rejected, with the rejection reason encoded in the response text. +### Alpha release restrictions +The alpha release does not have authentication, payments nor rate limiting, therefore some self imposed restrictions apply: + +- `start_time` should be within the next 6 blocks `[current_time+1, current_time+6]` +- `end_time` cannot be bigger than (roughtly) a month. That is `4320 blocks` + #### Usage From 1a91a68d52f40e3a04535dab9093852ddb02519f Mon Sep 17 00:00:00 2001 From: Salvatore Ingala <6681844+bigspider@users.noreply.github.com> Date: Fri, 14 Feb 2020 15:21:58 +0000 Subject: [PATCH 22/33] typo, bash_rc ==> bashrc --- apps/cli/INSTALL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/cli/INSTALL.md b/apps/cli/INSTALL.md index 80068e0..6eca6d1 100644 --- a/apps/cli/INSTALL.md +++ b/apps/cli/INSTALL.md @@ -13,9 +13,9 @@ For example, for user alice running a UNIX system and having `apps` in her home export PYTHONPATH=$PYTHONPATH:/home/alice/ -You should also include the command in your `.bash_rc` to avoid having to run it every time you open a new terminal. You can do it by running: +You should also include the command in your `.bashrc` to avoid having to run it every time you open a new terminal. You can do it by running: - echo 'export PYTHONPATH=$PYTHONPATH:' >> ~/.bash_rc + echo 'export PYTHONPATH=$PYTHONPATH:' >> ~/.bashrc Once the `PYTHONPATH` is set, you should be able to run `wt_cli` straightaway. Try it by running: @@ -32,4 +32,4 @@ Notice that if you run `wt_cli` as a module, you'll need to replace all the call ## Modify configuration parameters If you'd like to modify some of the configuration defaults (such as the user directory, where the logs and appointment receipts will be stored) you can do so in the config file located at: - /apps/cli/conf.py \ No newline at end of file + /apps/cli/conf.py From 244c04cb0de5cb13883388887286095faa360997 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 16:31:04 +0100 Subject: [PATCH 23/33] Fixes some typos --- apps/cli/INSTALL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/cli/INSTALL.md b/apps/cli/INSTALL.md index 6eca6d1..f831f83 100644 --- a/apps/cli/INSTALL.md +++ b/apps/cli/INSTALL.md @@ -2,7 +2,7 @@ `wt_cli` has some dependencies that can be satisfied by following [DEPENDENCIES.md](DEPENDENCIES.md). If your system already satisfies the dependencies, you can skip that part. -There are two ways of running `wt_cli`: adding the library to the `PYTHONPATH` env variable, or running it as a module +There are two ways of running `wt_cli`: adding the library to the `PYTHONPATH` env variable, or running it as a module. ## Modifying `PYTHONPATH` In order to run `wt_cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `apps` folder. You can do so by running: @@ -23,11 +23,11 @@ Once the `PYTHONPATH` is set, you should be able to run `wt_cli` straightaway. T python wt_cli.py -h ## Running `wt_cli` as a module -Python code can be also run as a module, to do you need to use `python -m`. From `apps` **parent** directory run: +Python code can be also run as a module, to do so you need to use `python -m`. From `apps` **parent** directory run: python -m apps.cli.wt_cli -h -Notice that if you run `wt_cli` as a module, you'll need to replace all the calls to `python wt_cli.py ` to `python -m apps.cli.wt_cli ` +Notice that if you run `wt_cli` as a module, you'll need to replace all the calls from `python wt_cli.py ` to `python -m apps.cli.wt_cli ` ## Modify configuration parameters If you'd like to modify some of the configuration defaults (such as the user directory, where the logs and appointment receipts will be stored) you can do so in the config file located at: From e9a9ccf440b36c4053716c0878e197a936373521 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 16:40:02 +0100 Subject: [PATCH 24/33] Fixes punctuation and typos on cli/README --- apps/cli/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/apps/cli/README.md b/apps/cli/README.md index 3ce7472..82af535 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -15,9 +15,8 @@ Refer to [INSTALL.md](INSTALL.md) #### Global options -- `-s, --server`: API server where to send the requests. Defaults to btc.pisa.watch (modifiable in \_\_init\_\_.py) -- `-p, --port` : API port where to send the requests. Defaults to 9814 (modifiable in \_\_init\_\_.py) -- `-d, --debug`: shows debug information and stores it in pisa.log +- `-s, --server`: API server where to send the requests. Defaults to https://teos.pisa.watch (modifiable in conf.py) +- `-p, --port` : API port where to send the requests. Defaults to 443 (modifiable in conf.py) - `-h --help`: shows a list of commands or help for a specific command. #### Commands @@ -53,8 +52,8 @@ The API will return a `application/json` HTTP response code `200/OK` if the appo ### Alpha release restrictions The alpha release does not have authentication, payments nor rate limiting, therefore some self imposed restrictions apply: -- `start_time` should be within the next 6 blocks `[current_time+1, current_time+6]` -- `end_time` cannot be bigger than (roughtly) a month. That is `4320 blocks` +- `start_time` should be within the next 6 blocks `[current_time+1, current_time+6]`. +- `end_time` cannot be bigger than (roughtly) a month. That is `4320` blocks on top of `start_time`. #### Usage @@ -70,7 +69,7 @@ if `-f, --file` **is** specified, then the command expects a path to a json file This command is used to get information about an specific appointment from the PISA server. -**Appointment can be in three states** +**Appointment can be in three states:** - `not_found`: meaning the locator is not recognised by the tower. This can either mean the locator is wrong, or the appointment has already been fulfilled (the PISA server does not keep track of completed appointments for now). - `being_watched`: the appointment has been accepted by the PISA server and it's being watched at the moment. This stage means that the dispute transaction has not been seen yet, and therefore no penalty transaction has been broadcast. @@ -144,4 +143,4 @@ or ## PISA API -If you wish to read about the underlying API, and how to write your own tool to interact with it, refer to [PISA-API.md](PISA-API.md) +If you wish to read about the underlying API, and how to write your own tool to interact with it, refer to [PISA-API.md](PISA-API.md). From e1f14fddc119e7e51c91c5bad9f63bbebc8bf7f4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 14 Feb 2020 16:47:25 +0100 Subject: [PATCH 25/33] Fixes typos and updates variable names in cli/PISA_API --- apps/cli/PISA-API.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/cli/PISA-API.md b/apps/cli/PISA-API.md index bf52edf..b843816 100644 --- a/apps/cli/PISA-API.md +++ b/apps/cli/PISA-API.md @@ -7,7 +7,7 @@ The PISA REST API consists, currently, of two endpoints: `/` and `/get_appointme `/` is the default endpoint, and is where the appointments should be sent to. `/` accepts `HTTP POST` requests only, with json request body, where data must match the following format: {"locator": l, "start_time": s, "end_time": e, - "dispute_delta": d, "encrypted_blob": eb} + "to_self_delay": d, "encrypted_blob": eb} We'll discuss the parameters one by one in the following: @@ -21,8 +21,8 @@ The to\_self\_delay, `d`, is the time PISA would have to respond with the **pena The encrypted\_blob, `eb`, is a data blob containing the `raw penalty transaction` and it is encrypted using `CHACHA20-POLY1305`. The `encryption key` used by the cipher is the sha256 of the **dispute transaction id**, and the `nonce` is a 12-byte long zero byte array: - sk = sk = sha256(unhexlify(secret)).digest() - nonce = nonce = bytearray(12) # b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + sk = sha256(unhexlify(secret)).digest() + nonce = bytearray(12) # b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' Finally, the encrypted blob must be hex encoded. `type(eb) = hex encoded str` @@ -38,11 +38,11 @@ The API will return a `application/json` HTTP response code `200/OK` if the appo # Get appointment -`/get_appointment` is an endpoint provided to check the status of the appointments sent to PISA. The endpoint is accessible without any type of authentication for now. `/get_appointment` accepts `HTTP GET` requests only, where the data to be provided must be the locator of an appointment. The query must match the following format: +`/get_appointment` is an endpoint provided to check the status of the appointments sent to PISA. The endpoint is accessible without any type of authentication for now. `/get_appointment` accepts `HTTP GET` requests only, where the data to be provided must be the **locator** of an appointment. The query must match the following format: `https://pisa_server:pisa_port/get_appointment?locator=appointment_locator` -### Appointment can be in three states +**Appointment can be in three states**: - `not_found`: meaning the locator is not recognised by the API. This could either mean the locator is wrong, or the appointment has already been fulfilled. - `being_watched`: the appointment has been accepted by the PISA server and it's being watched at the moment. This stage means that the dispute transaction has not been seen yet, and therefore no penalty transaction has been published. From 6e737b735c4278c15a60b948dbc1138ad3ff4779 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Sat, 15 Feb 2020 19:24:20 +0100 Subject: [PATCH 26/33] Logs real IP if API is behind a reverse proxy --- pisa/api.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pisa/api.py b/pisa/api.py index efb519a..ffd7c0e 100644 --- a/pisa/api.py +++ b/pisa/api.py @@ -35,10 +35,12 @@ class API: can be found at :mod:`Errors `. """ - remote_addr = request.environ.get("REMOTE_ADDR") - remote_port = request.environ.get("REMOTE_PORT") + # Getting the real IP if the server is behind a reverse proxy + remote_addr = request.environ.get("HTTP_X_REAL_IP") + if not remote_addr: + remote_addr = request.environ.get("REMOTE_ADDR") - logger.info("Received add_appointment request", from_addr_port="{}:{}".format(remote_addr, remote_port)) + logger.info("Received add_appointment request", from_addr="{}".format(remote_addr)) if request.is_json: # Check content type once if properly defined @@ -77,10 +79,7 @@ class API: response = None logger.info( - "Sending response and disconnecting", - from_addr_port="{}:{}".format(remote_addr, remote_port), - response=response, - error=error, + "Sending response and disconnecting", from_addr="{}".format(remote_addr), response=response, error=error ) if error is None: @@ -106,15 +105,16 @@ class API: - Appointments hold by the :obj:`Responder ` are flagged as ``dispute_triggered``. - Unknown appointments are flagged as ``not_found``. """ - remote_addr = request.environ.get("REMOTE_ADDR") - remote_port = request.environ.get("REMOTE_PORT") + + # Getting the real IP if the server is behind a reverse proxy + remote_addr = request.environ.get("HTTP_X_REAL_IP") + if not remote_addr: + remote_addr = request.environ.get("REMOTE_ADDR") locator = request.args.get("locator") response = [] - logger.info( - "Received get_appointment request", from_addr_port="{}:{}".format(remote_addr, remote_port), locator=locator - ) + logger.info("Received get_appointment request", from_addr="{}".format(remote_addr), locator=locator) # ToDo: #15-add-system-monitor if not isinstance(locator, str) or len(locator) != LOCATOR_LEN_HEX: From 4406beb5cd50a5ffd46b3121a7db28d93db48717 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Sun, 16 Feb 2020 15:59:04 +0100 Subject: [PATCH 27/33] Limits the encrypted_blob size to 2Kib for the alpha --- pisa/inspector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pisa/inspector.py b/pisa/inspector.py index 2c135e7..18cd899 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -20,6 +20,7 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_ BLOCKS_IN_A_MONTH = 4320 # 4320 = roughly a month in blocks +ENCRYPTED_BLOB_MAX_SIZE_HEX = 2 * 2048 class Inspector: @@ -321,6 +322,10 @@ class Inspector: rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE message = "wrong encrypted_blob data type ({})".format(t) + elif len(encrypted_blob) > ENCRYPTED_BLOB_MAX_SIZE_HEX: + rcode = errors.APPOINTMENT_FIELD_TOO_BIG + message = "encrypted_blob has to be 2Kib at most (current {})".format(len(encrypted_blob) // 2) + elif re.search(r"^[0-9A-Fa-f]+$", encrypted_blob) is None: rcode = errors.APPOINTMENT_WRONG_FIELD_FORMAT message = "wrong encrypted_blob format ({})".format(encrypted_blob) From fc24a4e8c96ebc9a19324c2f093881ce8eed9a17 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Sun, 16 Feb 2020 17:42:10 +0100 Subject: [PATCH 28/33] Fixes start_time range log --- pisa/inspector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pisa/inspector.py b/pisa/inspector.py index 18cd899..44ada80 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -167,7 +167,7 @@ class Inspector: else: message = ( "start_time is too close to current height. " - "Accepted times are: [current_height+1, current_height+2]" + "Accepted times are: [current_height+1, current_height+6]" ) elif start_time > block_height + 6: From 605726751a0d76f40fa449e8035afed54fc80370 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Sun, 16 Feb 2020 17:55:40 +0100 Subject: [PATCH 29/33] Updates README with 2kib limitation for encrypted_blob --- apps/cli/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/cli/README.md b/apps/cli/README.md index 82af535..0e5c825 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -54,6 +54,7 @@ The alpha release does not have authentication, payments nor rate limiting, ther - `start_time` should be within the next 6 blocks `[current_time+1, current_time+6]`. - `end_time` cannot be bigger than (roughtly) a month. That is `4320` blocks on top of `start_time`. +- `encrypted_blob`s are limited to `2 kib`. #### Usage From f6d95514aa6de734b77feefa5a573319c7052040 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Sun, 16 Feb 2020 17:57:44 +0100 Subject: [PATCH 30/33] Adds alpha release restrictions to PISA_API.md --- apps/cli/PISA-API.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/cli/PISA-API.md b/apps/cli/PISA-API.md index b843816..a95697f 100644 --- a/apps/cli/PISA-API.md +++ b/apps/cli/PISA-API.md @@ -26,7 +26,14 @@ The encrypted\_blob, `eb`, is a data blob containing the `raw penalty transactio Finally, the encrypted blob must be hex encoded. `type(eb) = hex encoded str` -The API will return a `application/json` HTTP response code `200/OK` if the appointment is accepted, with the locator encoded in the response text, or a `400/Bad Request` if the appointment is rejected, with the rejection reason encoded in the response text. +The API will return a `application/json` HTTP response code `200/OK` if the appointment is accepted, with the locator encoded in the response text, or a `400/Bad Request` if the appointment is rejected, with the rejection reason encoded in the response text. + +### Alpha release restrictions +The alpha release does not have authentication, payments nor rate limiting, therefore some self imposed restrictions apply: + +- `start_time` should be within the next 6 blocks `[current_time+1, current_time+6]`. +- `end_time` cannot be bigger than (roughtly) a month. That is `4320` blocks on top of `start_time`. +- `encrypted_blob`s are limited to `2 kib`. #### Appointment example From 5b32befade963af94d1babf8f05bb4f8176baef9 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 17 Feb 2020 10:41:02 +0100 Subject: [PATCH 31/33] Removes logging txids list from every block The list of transactions can be queried afterwards and it is currently flooding the log files. I may consider putting it back once we have a log parser, but it seems pretty unnecessary. --- pisa/responder.py | 1 - pisa/watcher.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pisa/responder.py b/pisa/responder.py index 4ae73ab..8be3fd8 100644 --- a/pisa/responder.py +++ b/pisa/responder.py @@ -275,7 +275,6 @@ class Responder: if len(self.trackers) > 0 and block is not None: txids = block.get("tx") - logger.info("List of transactions", txids=txids) if self.last_known_block == block.get("previousblockhash"): self.check_confirmations(txids) diff --git a/pisa/watcher.py b/pisa/watcher.py index c06bee3..2efca07 100644 --- a/pisa/watcher.py +++ b/pisa/watcher.py @@ -144,7 +144,6 @@ class Watcher: if len(self.appointments) > 0 and block is not None: txids = block.get("tx") - logger.info("List of transactions", txids=txids) expired_appointments = [ uuid From defe946b2c67acb586893e08971300a372689f3b Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 17 Feb 2020 11:12:18 +0100 Subject: [PATCH 32/33] Adds a more verbose return for server errors --- apps/cli/wt_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/cli/wt_cli.py b/apps/cli/wt_cli.py index 6a08b3b..0f10733 100644 --- a/apps/cli/wt_cli.py +++ b/apps/cli/wt_cli.py @@ -269,7 +269,9 @@ def process_post_appointment_response(response): response_json = response.json() except json.JSONDecodeError: - logger.error("The response was not valid JSON") + logger.error( + "The server returned a non-JSON response", status_code=response.status_code, reason=response.reason + ) return None if response.status_code != constants.HTTP_OK: From 07aa9a73ee3366f8c0120e6871548fcb4ec090bb Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 17 Feb 2020 12:21:53 +0100 Subject: [PATCH 33/33] Adds debug logging of all request to track possible bugs (stored in file only) --- pisa/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pisa/api.py b/pisa/api.py index ffd7c0e..c7b4e20 100644 --- a/pisa/api.py +++ b/pisa/api.py @@ -42,6 +42,9 @@ class API: logger.info("Received add_appointment request", from_addr="{}".format(remote_addr)) + # FIXME: Logging every request so we can get better understanding of bugs in the alpha + logger.debug("Request details", data="{}".format(request.data)) + if request.is_json: # Check content type once if properly defined request_data = json.loads(request.get_json())