diff --git a/.circleci/config.yml b/.circleci/config.yml index 32cf538..863cebb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: # Download and cache dependencies - restore_cache: keys: - - v1-dependencies-{{ checksum "teos/requirements.txt" }}-{{ checksum "teos/requirements-dev.txt" }}-{{ checksum "apps/cli/requirements-dev.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }} + - v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "cli/requirements.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }} - run: name: Install dependencies @@ -31,15 +31,15 @@ jobs: python3 -m venv venv . venv/bin/activate sudo pip install --upgrade pip - pip install -r teos/requirements.txt - pip install -r teos/requirements-dev.txt - pip install -r apps/cli/requirements-dev.txt + pip install -r requirements.txt + pip install -r requirements-dev.txt + pip install -r cli/requirements.txt - save_cache: paths: - ./venv - - /snap - key: v1-dependencies-{{ checksum "teos/requirements.txt" }}-{{ checksum "teos/requirements-dev.txt" }}-{{ checksum "apps/cli/requirements-dev.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }} + #- /snap + key: v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "cli/requirements.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }} # Run bitcoind for E2E testing (running it early so it has time to bootstrap) - run: @@ -50,12 +50,6 @@ jobs: /snap/bin/bitcoin-core.daemon # Run unit tests - - run: - name: Creates config files - command: | - cp teos/sample_conf.py teos/conf.py - cp apps/cli/sample_conf.py apps/cli/conf.py - - run: name: Run teos unit tests command: | @@ -72,7 +66,7 @@ jobs: name: Run cli unit tests command: | . venv/bin/activate - pytest test/apps/cli/unit + pytest test/cli/unit # Setup teos for E2E testing - run: @@ -80,8 +74,8 @@ jobs: command: | . venv/bin/activate cp test/teos/e2e/teos-conf.py teos/conf.py - python3 -m apps.generate_key -d ~/.teos/ - python3 -m apps.generate_key -n cli -d ~/.teos_cli/ + python3 -m generate_keys -d ~/.teos/ + python3 -m generate_keys -n cli -d ~/.teos_cli/ cp ~/.teos/teos_pk.der ~/.teos_cli/ diff --git a/.gitignore b/.gitignore index 9dac4f1..64b00b0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,11 +3,10 @@ .venv/ *.log .DS_Store -conf.py bitcoin.conf* *__pycache__ .pending* -apps/cli/*.json +cli/*.json appointments/ test.py *.pyc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a25fd79..2a13c1b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to The Eye of Satoshi +# Contributing to the Eye of Satoshi The following is a set of guidelines for contributing to TEOS. @@ -46,6 +46,10 @@ if rcode == 0: rcode, message = self.check_end_time(end_time, start_time, block_height) ``` +## Dev Requirements +In order to contrubite you will need to install additional dependencies. They can be found at `requirements-dev.txt`. Install them by running: + + pip install -r requirements-dev.txt ## Code Documentation Code should be, at least, documented using docstrings. We use the [Sphinx Google Style](https://www.sphinx-doc.org/en/master/usage/extensions/example_google.html#example-google) for documenting functions. diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md new file mode 100644 index 0000000..4f9e611 --- /dev/null +++ b/DEPENDENCIES.md @@ -0,0 +1,122 @@ +# Dependencies + +`teos` has both system-wide and Python dependencies. This document walks you through how to satisfy them. + +## System-wide Dependencies + +`teos` has the following system-wide dependencies: + +- `python3` +- `pip3` +- `bitcoind` + +### Checking if the dependencies are already satisfied + +Most UNIX systems ship with `python3` already installed, whereas OSX systems tend to ship with `python2`. In order to check our python version we should run: + + python --version + +For what we will get something like: + + Python 2.X.X + +Or + + Python 3.X.X + +It is also likely that, if `python3` is installed in our system, the `python` alias is not set to it but instead to `python2`. In order to check so, we can run: + + python3 --version + +If `python3` is installed but the `python` alias is not set to it, we should either set it, or use `python3` to run `teos`. + +Regarding `pip`, we can check what version is installed in our system (if any) by running: + + pip --version + +For what we will get something like: + + pip X.X.X from /usr/local/lib/python2.X/dist-packages/pip (python 2.X) + +Or + + pip X.X.X from /usr/local/lib/python3.X/dist-packages/pip (python 3.X) + +A similar thing to the `python` alias applies to the `pip` alias. We can check if pip3 is install by running: + + pip3 --version + +And, if it happens to be installed, change the alias to `pip3`, or use `pip3` instead of `pip`. + +### Installing bitcoind + +`teos` runs on top of a Bitcoin Core node. Other underlaying Bitcoin nodes are not supported at the moment. + +You can get Bitcoin Core from [bitcoin.org](https://bitcoin.org/en/download). + +Bitcoin needs to be running with the following options enables: + +- `zmq` for rawblockhash notifications +- `txindex` to be able to look for non-wallet transactions +- `server` to run rpc commands + +Here's an example of a `bitcoin.conf` you can use: + +``` +# [debug] +regtest=1 +daemon=1 +debug=1 +logips=1 + +# [rpc] +server=1 +rpcuser=user +rpcpassword=passwd + +# [blockchain] +txindex=1 + +# [ZMQ] +zmqpubhashblock=tcp://127.0.0.1:28332 +zmqpubrawblock=tcp://127.0.0.1:28332 +zmqpubhashtx=tcp://127.0.0.1:28333 +zmqpubrawtx=tcp://127.0.0.1:28333 +``` + +### Installing the Dependencies + +`python3` ca be downloaded from the [Python official website](https://www.python.org/downloads/) or installed using a package manager, depending on your distribution. Examples for both UNIX-like and OSX systems are provided. + +#### Ubuntu + +`python3` can be installed using `apt` as follows: + + sudo apt-get update + sudo apt-get install python3 + +and for `pip3`: + + sudo apt-get install python3-pip + pip install --upgrade pip==9.0.3 + +#### OSX + +`python3` can be installed using `Homebrew` as follows: + + brew install python3 + +`pip3` will be installed alongside `python3` in this case. + +## Python Dependencies + +`teos` has the following dependencies (which can be satisfied by using `pip install -r requirements.txt`): + +- `zmq` +- `flask` +- `cryptography` +- `coincurve` +- `pyzbase32` +- `requests` +- `plyvel` + diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..26cdad5 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,30 @@ +# Install + +`teos` has some dependencies that can be satisfied by following [DEPENDENCIES.md](DEPENDENCIES.md). If your system already satisfies the dependencies, you can skip that part. + +There are two ways of running `teos`: running it as a module or adding the library to the `PYTHONPATH` env variable. + +## Running `teos` as a Module +The easiest way to run `teos` is as a module. To do so you need to use `python -m`. From the teos parent directory run: + + python -m teos.teosd + +Notice that if you run `teos` as a module, you'll need to replace all the calls from `python teosd.py` to `python -m teos.teosd` + +## Modifying `PYTHONPATH` +Alternatively, you can add `teos` to your `PYTHONPATH`. You can do so by running: + + export PYTHONPATH=$PYTHONPATH: + +For example, for user alice running a UNIX system and having `python-teos` in her home folder, she would run: + + export PYTHONPATH=$PYTHONPATH:/home/alice/python-teos/ + +You should also include the command in your `.bashrc` to avoid having to run it every time you open a new terminal. You can do it by running: + + echo 'export PYTHONPATH=$PYTHONPATH:' >> ~/.bashrc + +## Modify Configuration Parameters +If you'd like to modify some of the configuration defaults (such as the user directory, where the logs will be stored) you can do so in the config file located at: + + /teos/conf.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..60942b4 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +## State of the Code + +Currently working on updating the software to match [BOLT13 rev1](https://github.com/sr-gi/bolt13). + +# The Eye of Satoshi (TEOS) + +The Eye of Satoshi is a Lightning watchtower compliant with [BOLT13](https://github.com/sr-gi/bolt13), written in Python 3. + +`teos` consists in three main modules: + +- `teos`: including the tower's main functionality (server-side) +- `cli`: including a reference command line interface (client-side) +- `common`: including shared functionality between `teos` and `cli`. + +Additionally, tests for every module can be found at `tests`. + +By default, `teos` will run on `regtest`. In order to run it on another network you need to change your `bitcoin.conf` (to run in the proper network) and your `conf.py` to match the network name and rpc port: + +``` +BTC_RPC_PORT = 18443 +BTC_NETWORK = "regtest" +``` + +### Running TEOS +You can run `teos` buy running `teosd.py` under `teos`. + +`teos` comes with a default configuration file (check [conf.py](teos/conf.py)). The configuration file include, amongst others, where your data folder is placed, what network it connects to, etc. + +To run `teos` you need a set of keys (to sign appointments) stored in your data directory. You can follow [generate keys](#generate-keys) to generate them. + +### Interacting with a TEOS Instance +You can interact with a `teos` instance (either run by yourself or someone else) by using `teos_cli` under `cli`. + +Since `teos_cli` works independently of `teos`, it uses a different configuration file (check [cli/conf.py](cli/conf.py)). + +`teos_cli` also needs an independent set of keys (that can be generated following [generate keys](#generate-keys)) as well as the public key of the tower instance (`teos_pk.der`). The same data directory can be used for both if you are running things locally. + +Notice that `teos_cli` is a simple way to interact with `teos`, but ideally that should be part of your wallet functionality (therefore why they are independent entities). `teos_cli` can be used as an example for how to send data to a [BOLT13](https://github.com/sr-gi/bolt13) compliant watchtower. + +### Generate Keys + +In order to generate a pair of keys for `teos` (or `teos_cli`) you can use `generate_keys.py`. + +The script generates a set of keys (`teos_sk.der` and `teos_pk.der`) in the current directory, by default. The name and output directory can be changed using `-n` and `-d` respectively. + +The following command will generate a set of keys for `teos` and store it in the default data directory (`~/.teos`): +``` +python generate_keys.py -d ~./teos +``` + +The following command will generate a set of keys for `teos_cli` and store it in the default data directory (`~/.teos_cli`): +``` +python generate_keys.py -n cli -d ~./teos_cli +``` + +Notice that `cli` needs a copy of the tower public key, so you should make a copy of that if you're using different data directories (as in this example): + +``` +cp ~./teos/teos_pk.der ~./teos_cli/teos_pk.der +``` + +## Dependencies +Refer to [DEPENDENCIES.md](DEPENDENCIES.md) + +## Installation + +Refer to [INSTALL.md](INSTALL.md) + +## Contributing +Refer to [CONTRIBUTING.md](CONTRIBUTING.md) \ No newline at end of file diff --git a/apps/cli/requirements-dev.txt b/apps/cli/requirements-dev.txt deleted file mode 100644 index f8c47f0..0000000 --- a/apps/cli/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ -responses -pytest -black \ No newline at end of file diff --git a/apps/cli/DEPENDENCIES.md b/cli/DEPENDENCIES.md similarity index 100% rename from apps/cli/DEPENDENCIES.md rename to cli/DEPENDENCIES.md diff --git a/apps/cli/INSTALL.md b/cli/INSTALL.md similarity index 69% rename from apps/cli/INSTALL.md rename to cli/INSTALL.md index a5a32d5..2c9d4bf 100644 --- a/apps/cli/INSTALL.md +++ b/cli/INSTALL.md @@ -5,31 +5,31 @@ There are two ways of running `teos_cli`: adding the library to the `PYTHONPATH` env variable, or running it as a module. ## Modifying `PYTHONPATH` -In order to run `teos_cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `apps` folder. You can do so by running: +In order to run `teos_cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `cli` folder. You can do so by running: - export PYTHONPATH=$PYTHONPATH: + export PYTHONPATH=$PYTHONPATH: -For example, for user alice running a UNIX system and having `apps` in her home folder, she would run: +For example, for user alice running a UNIX system and having `cli` in her home folder, she would run: export PYTHONPATH=$PYTHONPATH:/home/alice/ You should also include the command in your `.bashrc` to avoid having to run it every time you open a new terminal. You can do it by running: - echo 'export PYTHONPATH=$PYTHONPATH:' >> ~/.bashrc + echo 'export PYTHONPATH=$PYTHONPATH:' >> ~/.bashrc Once the `PYTHONPATH` is set, you should be able to run `teos_cli` straightaway. Try it by running: - cd /apps/cli + cd /cli python teos_cli.py -h ## Running `teos_cli` as a module -Python code can be also run as a module, to do so you need to use `python -m`. From `apps` **parent** directory run: +Python code can be also run as a module, to do so you need to use `python -m`. From `cli` **parent** directory run: - python -m apps.cli.teos_cli -h + python -m cli.teos_cli -h -Notice that if you run `teos_cli` as a module, you'll need to replace all the calls from `python teos_cli.py ` to `python -m apps.cli.teos_cli ` +Notice that if you run `teos_cli` as a module, you'll need to replace all the calls from `python teos_cli.py ` to `python -m cli.teos_cli ` ## Modify configuration parameters If you'd like to modify some of the configuration defaults (such as the user directory, where the logs and appointment receipts will be stored) you can do so in the config file located at: - /apps/cli/conf.py + /cli/conf.py diff --git a/apps/cli/README.md b/cli/README.md similarity index 100% rename from apps/cli/README.md rename to cli/README.md diff --git a/apps/cli/TEOS-API.md b/cli/TEOS-API.md similarity index 100% rename from apps/cli/TEOS-API.md rename to cli/TEOS-API.md diff --git a/apps/cli/__init__.py b/cli/__init__.py similarity index 97% rename from apps/cli/__init__.py rename to cli/__init__.py index 76d2246..925027f 100644 --- a/apps/cli/__init__.py +++ b/cli/__init__.py @@ -1,5 +1,5 @@ import os -import apps.cli.conf as conf +import cli.conf as conf from common.tools import extend_paths, check_conf_fields, setup_logging, setup_data_folder LOG_PREFIX = "cli" diff --git a/apps/cli/sample_conf.py b/cli/conf.py similarity index 74% rename from apps/cli/sample_conf.py rename to cli/conf.py index 155e788..1072da7 100644 --- a/apps/cli/sample_conf.py +++ b/cli/conf.py @@ -1,6 +1,6 @@ # TEOS-SERVER -DEFAULT_TEOS_API_SERVER = "https://teos.pisa.watch" -DEFAULT_TEOS_API_PORT = 443 +DEFAULT_TEOS_API_SERVER = "http://localhost" +DEFAULT_TEOS_API_PORT = 9814 # WT-CLI DATA_FOLDER = "~/.teos_cli/" diff --git a/apps/cli/help.py b/cli/help.py similarity index 100% rename from apps/cli/help.py rename to cli/help.py diff --git a/apps/cli/requirements.txt b/cli/requirements.txt similarity index 100% rename from apps/cli/requirements.txt rename to cli/requirements.txt diff --git a/apps/cli/teos_cli.py b/cli/teos_cli.py similarity index 99% rename from apps/cli/teos_cli.py rename to cli/teos_cli.py index d1d15f0..bbfbd68 100644 --- a/apps/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -10,8 +10,8 @@ from coincurve import PublicKey from getopt import getopt, GetoptError from requests import ConnectTimeout, ConnectionError -from apps.cli import config, LOG_PREFIX -from apps.cli.help import help_add_appointment, help_get_appointment +from cli import config, LOG_PREFIX +from cli.help import help_add_appointment, help_get_appointment from common.blob import Blob import common.cryptographer diff --git a/apps/generate_key.py b/generate_keys.py similarity index 80% rename from apps/generate_key.py rename to generate_keys.py index 5a6b449..add59c7 100644 --- a/apps/generate_key.py +++ b/generate_keys.py @@ -7,8 +7,16 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import ec -# Simple tool to generate an ECDSA private key using the secp256k1 curve and save private and public keys -# as 'teos_sk.der' 'and teos_pk.der', respectively. +""" +Simple way too to generate an ECDSA key pair using the secp256k1 curve and store it on disk. + +Default output dir = . +Default key naming = teos_sk.der and teos_pk.der + +The output dir and naming can be changed using the -n and -d command line options, respectively. + +DISCLAIMER: Use this for testing only, using keys generated from this file in production is highly unadvised. +""" def save_sk(sk, filename): diff --git a/teos/requirements-dev.txt b/requirements-dev.txt similarity index 77% rename from teos/requirements-dev.txt rename to requirements-dev.txt index 10481e4..8a51da8 100644 --- a/teos/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ pytest black +responses bitcoind_mock===0.0.4 diff --git a/teos/requirements.txt b/requirements.txt similarity index 71% rename from teos/requirements.txt rename to requirements.txt index f168a98..5ea83ae 100644 --- a/teos/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ zmq flask -cryptography +cryptography==2.8 coincurve pyzbase32 requests diff --git a/teos/sample_conf.py b/teos/conf.py similarity index 100% rename from teos/sample_conf.py rename to teos/conf.py diff --git a/teos/tools.py b/teos/tools.py index d10d63c..53b85cc 100644 --- a/teos/tools.py +++ b/teos/tools.py @@ -37,7 +37,7 @@ def can_connect_to_bitcoind(): try: bitcoin_cli().help() - except (timeout, ConnectionRefusedError, JSONRPCException, HTTPException): + except (timeout, ConnectionRefusedError, JSONRPCException, HTTPException, OSError): can_connect = False return can_connect diff --git a/test/apps/cli/__init__.py b/test/apps/cli/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/apps/cli/unit/__init__.py b/test/apps/cli/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/__init__.py b/test/cli/__init__.py similarity index 100% rename from apps/__init__.py rename to test/cli/__init__.py diff --git a/test/apps/__init__.py b/test/cli/unit/__init__.py similarity index 100% rename from test/apps/__init__.py rename to test/cli/unit/__init__.py diff --git a/test/apps/cli/unit/conftest.py b/test/cli/unit/conftest.py similarity index 100% rename from test/apps/cli/unit/conftest.py rename to test/cli/unit/conftest.py diff --git a/test/apps/cli/unit/test_teos_cli.py b/test/cli/unit/test_teos_cli.py similarity index 98% rename from test/apps/cli/unit/test_teos_cli.py rename to test/cli/unit/test_teos_cli.py index 00c9c9d..850a99f 100644 --- a/test/apps/cli/unit/test_teos_cli.py +++ b/test/cli/unit/test_teos_cli.py @@ -11,8 +11,8 @@ from common.appointment import Appointment from common.cryptographer import Cryptographer from common.blob import Blob -import apps.cli.teos_cli as teos_cli -from test.apps.cli.unit.conftest import get_random_value_hex +import cli.teos_cli as teos_cli +from test.cli.unit.conftest import get_random_value_hex common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=teos_cli.LOG_PREFIX) diff --git a/test/teos/e2e/test_basic_e2e.py b/test/teos/e2e/test_basic_e2e.py index ed6fab4..c87090a 100644 --- a/test/teos/e2e/test_basic_e2e.py +++ b/test/teos/e2e/test_basic_e2e.py @@ -5,7 +5,7 @@ from riemann.tx import Tx from teos import HOST, PORT -from apps.cli import teos_cli +from cli import teos_cli from common.blob import Blob import common.cryptographer @@ -21,7 +21,7 @@ from test.teos.e2e.conftest import ( create_penalty_tx, run_teosd, ) -from apps.cli import config as cli_conf +from cli import config as cli_conf common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="")