Merge pull request #105 from sr-gi/road-to-oss

Road to OSS

.circleci/config.yml

@@ -21,7 +21,7 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v1-dependencies-{{ checksum "teos/requirements.txt" }}-{{ checksum "teos/requirements-dev.txt" }}-{{ checksum "apps/cli/requirements-dev.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }}
+            - v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "cli/requirements.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }}
 
       - run:
           name: Install dependencies
@@ -31,15 +31,15 @@ jobs:
             python3 -m venv venv
             . venv/bin/activate
             sudo pip install --upgrade pip
-            pip install -r teos/requirements.txt
-            pip install -r teos/requirements-dev.txt
-            pip install -r apps/cli/requirements-dev.txt
+            pip install -r requirements.txt
+            pip install -r requirements-dev.txt
+            pip install -r cli/requirements.txt
 
       - save_cache:
           paths:
             - ./venv
-            - /snap
-          key: v1-dependencies-{{ checksum "teos/requirements.txt" }}-{{ checksum "teos/requirements-dev.txt" }}-{{ checksum "apps/cli/requirements-dev.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }}
+            #- /snap
+          key: v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "cli/requirements.txt" }}-{{ checksum "test/teos/e2e/bitcoind_snap.sh" }}
 
       # Run bitcoind for E2E testing (running it early so it has time to bootstrap)
       - run:
@@ -50,12 +50,6 @@ jobs:
             /snap/bin/bitcoin-core.daemon
 
       # Run unit tests
-      - run:
-          name: Creates config files
-          command: |
-            cp teos/sample_conf.py teos/conf.py
-            cp apps/cli/sample_conf.py apps/cli/conf.py
-
       - run:
           name: Run teos unit tests
           command: |
@@ -72,7 +66,7 @@ jobs:
           name: Run cli unit tests
           command: |
             . venv/bin/activate
-            pytest test/apps/cli/unit
+            pytest test/cli/unit
 
       # Setup teos for E2E testing
       - run:
@@ -80,8 +74,8 @@ jobs:
           command: |
             . venv/bin/activate
             cp test/teos/e2e/teos-conf.py teos/conf.py
-            python3 -m apps.generate_key -d ~/.teos/
-            python3 -m apps.generate_key -n cli -d ~/.teos_cli/
+            python3 -m generate_keys -d ~/.teos/
+            python3 -m generate_keys -n cli -d ~/.teos_cli/
             cp ~/.teos/teos_pk.der ~/.teos_cli/
 
 

.gitignore

@@ -3,11 +3,10 @@
 .venv/
 *.log
 .DS_Store
-conf.py
 bitcoin.conf*
 *__pycache__
 .pending*
-apps/cli/*.json
+cli/*.json
 appointments/
 test.py
 *.pyc

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Contributing to The Eye of Satoshi
+# Contributing to the Eye of Satoshi
 
 The following is a set of guidelines for contributing to TEOS.
 
@@ -46,6 +46,10 @@ if rcode == 0:
     rcode, message = self.check_end_time(end_time, start_time, block_height)
 ```
 
+## Dev Requirements
+In order to contrubite you will need to install additional dependencies. They can be found at `requirements-dev.txt`. Install them by running:
+
+	pip install -r requirements-dev.txt
 
 ## Code Documentation
 Code should be, at least, documented using docstrings. We use the [Sphinx Google Style](https://www.sphinx-doc.org/en/master/usage/extensions/example_google.html#example-google) for documenting functions.

DEPENDENCIES.md

@@ -0,0 +1,122 @@
+# Dependencies
+
+`teos` has both system-wide and Python dependencies. This document walks you through how to satisfy them.
+
+## System-wide Dependencies
+
+`teos` has the following system-wide dependencies:
+
+- `python3`
+- `pip3`
+- `bitcoind`
+
+### Checking if the dependencies are already satisfied
+
+Most UNIX systems ship with `python3` already installed, whereas OSX systems tend to ship with `python2`. In order to check our python version we should run:
+
+	python --version
+
+For what we will get something like:
+
+	Python 2.X.X
+	
+Or
+	
+	Python 3.X.X
+	
+It is also likely that, if `python3` is installed in our system, the `python` alias is not set to it but instead to `python2`. In order to check so, we can run:
+
+	python3 --version
+
+If `python3` is installed but the `python` alias is not set to it, we should either set it, or use `python3` to run `teos`.
+
+Regarding `pip`, we can check what version is installed in our system (if any) by running:
+
+	pip --version
+
+For what we will get something like:
+
+	pip X.X.X from /usr/local/lib/python2.X/dist-packages/pip (python 2.X)
+	
+Or
+
+	pip X.X.X from /usr/local/lib/python3.X/dist-packages/pip (python 3.X)
+
+A similar thing to the `python` alias applies to the `pip` alias. We can check if pip3 is install by running:
+
+	pip3 --version
+	
+And, if it happens to be installed, change the alias to `pip3`, or use `pip3` instead of `pip`.
+
+### Installing bitcoind
+
+`teos` runs on top of a Bitcoin Core node. Other underlaying Bitcoin nodes are not supported at the moment. 
+
+You can get Bitcoin Core from [bitcoin.org](https://bitcoin.org/en/download).
+
+Bitcoin needs to be running with the following options enables:
+
+- `zmq` for rawblockhash notifications
+- `txindex` to be able to look for non-wallet transactions
+- `server` to run rpc commands
+
+Here's an example of a `bitcoin.conf` you can use:
+
+```
+# [debug]
+regtest=1
+daemon=1
+debug=1
+logips=1
+
+# [rpc]
+server=1
+rpcuser=user
+rpcpassword=passwd
+
+# [blockchain]
+txindex=1
+
+# [ZMQ]
+zmqpubhashblock=tcp://127.0.0.1:28332
+zmqpubrawblock=tcp://127.0.0.1:28332
+zmqpubhashtx=tcp://127.0.0.1:28333
+zmqpubrawtx=tcp://127.0.0.1:28333
+```
+
+### Installing the Dependencies
+
+`python3` ca be downloaded from the [Python official website](https://www.python.org/downloads/) or installed using a package manager, depending on your distribution. Examples for both UNIX-like and OSX systems are provided.
+
+#### Ubuntu
+
+`python3` can be installed using `apt` as follows:
+
+	sudo apt-get update
+	sudo apt-get install python3
+	
+and for `pip3`:
+
+	sudo apt-get install python3-pip
+	pip install --upgrade pip==9.0.3
+	
+#### OSX
+
+`python3` can be installed using `Homebrew` as follows:
+	
+	brew install python3
+
+`pip3` will be installed alongside `python3` in this case.
+
+## Python Dependencies
+
+`teos` has the following dependencies (which can be satisfied by using `pip install -r requirements.txt`):
+
+- `zmq`
+- `flask`
+- `cryptography`
+- `coincurve`
+- `pyzbase32`
+- `requests`
+- `plyvel`
+

INSTALL.md

@@ -0,0 +1,30 @@
+# Install 
+
+`teos` has some dependencies that can be satisfied by following [DEPENDENCIES.md](DEPENDENCIES.md). If your system already satisfies the dependencies, you can skip that part.
+
+There are two ways of running `teos`: running it as a module or adding the library to the `PYTHONPATH` env variable.
+
+## Running `teos` as a Module
+The easiest way to run `teos` is as a module. To do so you need to use `python -m`. From the teos parent directory run:
+
+    python -m teos.teosd
+    
+Notice that if you run `teos` as a module, you'll need to replace all the calls from `python teosd.py` to `python -m teos.teosd` 
+
+## Modifying `PYTHONPATH`
+Alternatively, you can add `teos` to your `PYTHONPATH`. You can do so by running:
+
+	export PYTHONPATH=$PYTHONPATH:<absolute_path_to_teos_parent>
+	
+For example, for user alice running a UNIX system and having `python-teos` in her home folder, she would run:
+	
+	export PYTHONPATH=$PYTHONPATH:/home/alice/python-teos/
+	
+You should also include the command in your `.bashrc` to avoid having to run it every time you open a new terminal. You can do it by running:
+
+	echo 'export PYTHONPATH=$PYTHONPATH:<absolute_path_to_teos_parent>' >> ~/.bashrc
+	
+## Modify Configuration Parameters
+If you'd like to modify some of the configuration defaults (such as the user directory, where the logs will be stored) you can do so in the config file located at:
+
+	 <absolute_path_to_teos_parent>/teos/conf.py

README.md

@@ -0,0 +1,70 @@
+## State of the Code
+
+Currently working on updating the software to match [BOLT13 rev1](https://github.com/sr-gi/bolt13).
+
+# The Eye of Satoshi (TEOS)
+
+The Eye of Satoshi is a Lightning watchtower compliant with [BOLT13](https://github.com/sr-gi/bolt13), written in Python 3.
+
+`teos` consists in three main modules:
+
+- `teos`: including the tower's main functionality (server-side)
+- `cli`: including a reference command line interface (client-side)
+- `common`: including shared functionality between `teos` and `cli`.
+
+Additionally, tests for every module can be found at `tests`.
+
+By default, `teos` will run on `regtest`. In order to run it on another network you need to change your `bitcoin.conf` (to run in the proper network) and your `conf.py` to match the network name and rpc port:
+
+```
+BTC_RPC_PORT = 18443
+BTC_NETWORK = "regtest"
+```
+
+### Running TEOS
+You can run `teos` buy running `teosd.py` under `teos`. 
+
+`teos` comes with a default configuration file (check [conf.py](teos/conf.py)). The configuration file include, amongst others, where your data folder is placed, what network it connects to, etc.
+
+To run `teos` you need a set of keys (to sign appointments) stored in your data directory. You can follow [generate keys](#generate-keys) to generate them.
+
+### Interacting with a TEOS Instance
+You can interact with a `teos` instance (either run by yourself or someone else) by using `teos_cli` under `cli`.
+
+Since `teos_cli` works independently of `teos`, it uses a different configuration file (check [cli/conf.py](cli/conf.py)).
+
+`teos_cli` also needs an independent set of keys (that can be generated following [generate keys](#generate-keys)) as well as the public key of the tower instance (`teos_pk.der`). The same data directory can be used for both if you are running things locally.
+
+Notice that `teos_cli` is a simple way to interact with `teos`, but ideally that should be part of your wallet functionality (therefore why they are independent entities). `teos_cli` can be used as an example for how to send data to a [BOLT13](https://github.com/sr-gi/bolt13) compliant watchtower.
+
+### Generate Keys
+
+In order to generate a pair of keys for `teos` (or `teos_cli`) you can use `generate_keys.py`. 
+
+The script generates a set of keys (`teos_sk.der` and `teos_pk.der`) in the current directory, by default. The name and output directory can be changed using `-n` and `-d` respectively.
+
+The following command will generate a set of keys for `teos` and store it in the default data directory (`~/.teos`):
+```
+python generate_keys.py -d ~./teos
+``` 
+
+The following command will generate a set of keys for `teos_cli` and store it in the default data directory (`~/.teos_cli`):
+```
+python generate_keys.py -n cli -d ~./teos_cli
+``` 
+
+Notice that `cli` needs a copy of the tower public key, so you should make a copy of that if you're using different data directories (as in this example):
+
+```
+cp ~./teos/teos_pk.der ~./teos_cli/teos_pk.der 
+```
+
+## Dependencies
+Refer to [DEPENDENCIES.md](DEPENDENCIES.md)
+
+## Installation
+
+Refer to [INSTALL.md](INSTALL.md)
+
+## Contributing 
+Refer to [CONTRIBUTING.md](CONTRIBUTING.md)

apps/cli/requirements-dev.txt

@@ -1,3 +0,0 @@
-responses
-pytest
-black

cli/INSTALL.md

@@ -5,31 +5,31 @@
 There are two ways of running `teos_cli`: adding the library to the `PYTHONPATH` env variable, or running it as a module.
 
 ## Modifying `PYTHONPATH`
-In order to run `teos_cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `apps` folder. You can do so by running:
+In order to run `teos_cli`, you should set your `PYTHONPATH` env variable to include the folder that contains the `cli` folder. You can do so by running:
 
-	export PYTHONPATH=$PYTHONPATH:<absolute_path_to_apps>
+	export PYTHONPATH=$PYTHONPATH:<absolute_path_to_cli_parent>
 	
-For example, for user alice running a UNIX system and having `apps` in her home folder, she would run:
+For example, for user alice running a UNIX system and having `cli` in her home folder, she would run:
 	
 	export PYTHONPATH=$PYTHONPATH:/home/alice/
 	
 You should also include the command in your `.bashrc` to avoid having to run it every time you open a new terminal. You can do it by running:
 
-	echo 'export PYTHONPATH=$PYTHONPATH:<absolute_path_to_apps>' >> ~/.bashrc
+	echo 'export PYTHONPATH=$PYTHONPATH:<absolute_path_to_cli_parent>' >> ~/.bashrc
 	
 Once the `PYTHONPATH` is set, you should be able to run `teos_cli` straightaway. Try it by running:
 
-	cd <absolute_path_to_apps>/apps/cli
+	cd <absolute_path_to_cli_parent>/cli
 	python teos_cli.py -h
 	
 ## Running `teos_cli` as a module
-Python code can be also run as a module, to do so you need to use `python -m`. From `apps` **parent** directory run:
+Python code can be also run as a module, to do so you need to use `python -m`. From `cli` **parent** directory run:
 
-    python -m apps.cli.teos_cli -h
+    python -m cli.teos_cli -h
     
-Notice that if you run `teos_cli` as a module, you'll need to replace all the calls from `python teos_cli.py <argument>` to `python -m apps.cli.teos_cli <argument>` 
+Notice that if you run `teos_cli` as a module, you'll need to replace all the calls from `python teos_cli.py <argument>` to `python -m cli.teos_cli <argument>` 
 
 ## Modify configuration parameters
 If you'd like to modify some of the configuration defaults (such as the user directory, where the logs and appointment receipts will be stored) you can do so in the config file located at:
 
-	 <absolute_path_to_apps>/apps/cli/conf.py
+	 <absolute_path_to_cli_parent>/cli/conf.py

cli/__init__.py

@@ -1,5 +1,5 @@
 import os
-import apps.cli.conf as conf
+import cli.conf as conf
 from common.tools import extend_paths, check_conf_fields, setup_logging, setup_data_folder
 
 LOG_PREFIX = "cli"

cli/conf.py

@@ -1,6 +1,6 @@
 # TEOS-SERVER
-DEFAULT_TEOS_API_SERVER = "https://teos.pisa.watch"
-DEFAULT_TEOS_API_PORT = 443
+DEFAULT_TEOS_API_SERVER = "http://localhost"
+DEFAULT_TEOS_API_PORT = 9814
 
 # WT-CLI
 DATA_FOLDER = "~/.teos_cli/"

cli/teos_cli.py

@@ -10,8 +10,8 @@ from coincurve import PublicKey
 from getopt import getopt, GetoptError
 from requests import ConnectTimeout, ConnectionError
 
-from apps.cli import config, LOG_PREFIX
-from apps.cli.help import help_add_appointment, help_get_appointment
+from cli import config, LOG_PREFIX
+from cli.help import help_add_appointment, help_get_appointment
 from common.blob import Blob
 
 import common.cryptographer

generate_keys.py

@@ -7,8 +7,16 @@ from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import ec
 
 
-# Simple tool to generate an ECDSA private key using the secp256k1 curve and save private and public keys
-# as 'teos_sk.der' 'and teos_pk.der', respectively.
+"""
+Simple way too to generate an ECDSA key pair using the secp256k1 curve and store it on disk.
+
+Default output dir = .
+Default key naming = teos_sk.der and teos_pk.der
+
+The output dir and naming can be changed using the -n and -d command line options, respectively.
+
+DISCLAIMER: Use this for testing only, using keys generated from this file in production is highly unadvised.
+"""
 
 
 def save_sk(sk, filename):

requirements-dev.txt

@@ -1,3 +1,4 @@
 pytest
 black
+responses
 bitcoind_mock===0.0.4

requirements.txt

@@ -1,6 +1,6 @@
 zmq
 flask
-cryptography
+cryptography==2.8
 coincurve
 pyzbase32
 requests

teos/tools.py

@@ -37,7 +37,7 @@ def can_connect_to_bitcoind():
 
     try:
         bitcoin_cli().help()
-    except (timeout, ConnectionRefusedError, JSONRPCException, HTTPException):
+    except (timeout, ConnectionRefusedError, JSONRPCException, HTTPException, OSError):
         can_connect = False
 
     return can_connect

test/cli/unit/test_teos_cli.py

@@ -11,8 +11,8 @@ from common.appointment import Appointment
 from common.cryptographer import Cryptographer
 
 from common.blob import Blob
-import apps.cli.teos_cli as teos_cli
-from test.apps.cli.unit.conftest import get_random_value_hex
+import cli.teos_cli as teos_cli
+from test.cli.unit.conftest import get_random_value_hex
 
 common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=teos_cli.LOG_PREFIX)
 

test/teos/e2e/test_basic_e2e.py

@@ -5,7 +5,7 @@ from riemann.tx import Tx
 
 
 from teos import HOST, PORT
-from apps.cli import teos_cli
+from cli import teos_cli
 from common.blob import Blob
 
 import common.cryptographer
@@ -21,7 +21,7 @@ from test.teos.e2e.conftest import (
     create_penalty_tx,
     run_teosd,
 )
-from apps.cli import config as cli_conf
+from cli import config as cli_conf
 
 common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="")