pisa -> teos

test/teos/e2e/bitcoin.conf

@@ -0,0 +1,28 @@
+# [network]
+dnsseed=0
+
+# [debug]
+daemon=1
+regtest=1
+debug=1
+logips=1
+
+# [rpc]
+server=1
+rpcuser=user
+rpcpassword=passwd
+rpcallowip=0.0.0.0/0
+rpcservertimeout=300
+
+# [zmq]
+zmqpubhashblock=tcp://0.0.0.0:28335
+zmqpubrawblock=tcp://0.0.0.0:28335
+zmqpubrawtx=tcp://0.0.0.0:28336
+
+# [blockchain]
+txindex=1
+
+# There are some parameters that only work in the specific on regtest if specified in the regtest section
+[regtest]
+rpcbind=0.0.0.0
+rpcport=18445

test/teos/e2e/bitcoind_snap.sh

@@ -0,0 +1 @@
+bitcoin-core --channel=0.19/stable

test/teos/e2e/conftest.py

@@ -0,0 +1,118 @@
+import pytest
+import random
+from multiprocessing import Process
+from decimal import Decimal, getcontext
+
+import teos.conf as conf
+from teos.teosd import main
+from teos.utils.auth_proxy import AuthServiceProxy
+
+getcontext().prec = 10
+END_TIME_DELTA = 10
+
+
+@pytest.fixture(scope="session")
+def bitcoin_cli():
+    # return AuthServiceProxy("http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, 18444))
+    return AuthServiceProxy(
+        "http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, conf.BTC_RPC_PORT)
+    )
+
+
+@pytest.fixture(scope="session", autouse=True)
+def prng_seed():
+    random.seed(0)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_node(bitcoin_cli):
+    # This method will create a new address a mine bitcoin so the node can be used for testing
+    new_addr = bitcoin_cli.getnewaddress()
+    bitcoin_cli.generatetoaddress(101, new_addr)
+
+
+@pytest.fixture()
+def create_txs(bitcoin_cli):
+    utxos = bitcoin_cli.listunspent()
+
+    if len(utxos) == 0:
+        raise ValueError("There're no UTXOs.")
+
+    utxo = utxos.pop(0)
+    while utxo.get("amount") < Decimal(2 / pow(10, 5)):
+        utxo = utxos.pop(0)
+
+    signed_commitment_tx = create_commitment_tx(bitcoin_cli, utxo)
+    decoded_commitment_tx = bitcoin_cli.decoderawtransaction(signed_commitment_tx)
+
+    signed_penalty_tx = create_penalty_tx(bitcoin_cli, decoded_commitment_tx)
+
+    return signed_commitment_tx, signed_penalty_tx
+
+
+def run_teosd():
+    teosd_process = Process(target=main, daemon=True)
+    teosd_process.start()
+
+    return teosd_process
+
+
+def get_random_value_hex(nbytes):
+    pseudo_random_value = random.getrandbits(8 * nbytes)
+    prv_hex = "{:x}".format(pseudo_random_value)
+    return prv_hex.zfill(2 * nbytes)
+
+
+def create_commitment_tx(bitcoin_cli, utxo, destination=None):
+    # We will set the recipient to ourselves is destination is None
+    if destination is None:
+        destination = utxo.get("address")
+
+    commitment_tx_ins = {"txid": utxo.get("txid"), "vout": utxo.get("vout")}
+    commitment_tx_outs = {destination: utxo.get("amount") - Decimal(1 / pow(10, 5))}
+
+    raw_commitment_tx = bitcoin_cli.createrawtransaction([commitment_tx_ins], commitment_tx_outs)
+    signed_commitment_tx = bitcoin_cli.signrawtransactionwithwallet(raw_commitment_tx)
+
+    if not signed_commitment_tx.get("complete"):
+        raise ValueError("Couldn't sign transaction. {}".format(signed_commitment_tx))
+
+    return signed_commitment_tx.get("hex")
+
+
+def create_penalty_tx(bitcoin_cli, decoded_commitment_tx, destination=None):
+    # We will set the recipient to ourselves is destination is None
+    if destination is None:
+        destination = decoded_commitment_tx.get("vout")[0].get("scriptPubKey").get("addresses")[0]
+
+    penalty_tx_ins = {"txid": decoded_commitment_tx.get("txid"), "vout": 0}
+    penalty_tx_outs = {destination: decoded_commitment_tx.get("vout")[0].get("value") - Decimal(1 / pow(10, 5))}
+
+    orphan_info = {
+        "txid": decoded_commitment_tx.get("txid"),
+        "scriptPubKey": decoded_commitment_tx.get("vout")[0].get("scriptPubKey").get("hex"),
+        "vout": 0,
+        "amount": decoded_commitment_tx.get("vout")[0].get("value"),
+    }
+
+    raw_penalty_tx = bitcoin_cli.createrawtransaction([penalty_tx_ins], penalty_tx_outs)
+    signed_penalty_tx = bitcoin_cli.signrawtransactionwithwallet(raw_penalty_tx, [orphan_info])
+
+    if not signed_penalty_tx.get("complete"):
+        raise ValueError("Couldn't sign orphan transaction. {}".format(signed_penalty_tx))
+
+    return signed_penalty_tx.get("hex")
+
+
+def build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx):
+    current_height = bitcoin_cli.getblockcount()
+
+    appointment_data = {
+        "tx": penalty_tx,
+        "tx_id": commitment_tx_id,
+        "start_time": current_height + 1,
+        "end_time": current_height + 1 + END_TIME_DELTA,
+        "to_self_delay": 20,
+    }
+
+    return appointment_data

test/teos/e2e/teos-conf.py

@@ -0,0 +1,26 @@
+# bitcoind
+BTC_RPC_USER = "user"
+BTC_RPC_PASSWD = "passwd"
+BTC_RPC_HOST = "localhost"
+BTC_RPC_PORT = 18445
+BTC_NETWORK = "regtest"
+
+# ZMQ
+FEED_PROTOCOL = "tcp"
+FEED_ADDR = "127.0.0.1"
+FEED_PORT = 28335
+
+# TEOS
+DATA_FOLDER = "~/.teos/"
+MAX_APPOINTMENTS = 100
+EXPIRY_DELTA = 6
+MIN_TO_SELF_DELAY = 20
+SERVER_LOG_FILE = "teos.log"
+TEOS_SECRET_KEY = "teos_sk.der"
+
+# CHAIN MONITOR
+POLLING_DELTA = 60
+BLOCK_WINDOW_SIZE = 10
+
+# LEVELDB
+DB_PATH = "appointments"

test/teos/e2e/test_basic_e2e.py

@@ -0,0 +1,325 @@
+import json
+from time import sleep
+from riemann.tx import Tx
+
+from teos import config
+from teos import HOST, PORT
+from apps.cli import teos_cli
+from common.blob import Blob
+
+import common.cryptographer
+from common.logger import Logger
+from common.tools import compute_locator
+from common.appointment import Appointment
+from common.cryptographer import Cryptographer
+from teos.utils.auth_proxy import JSONRPCException
+from test.teos.e2e.conftest import (
+    END_TIME_DELTA,
+    build_appointment_data,
+    get_random_value_hex,
+    create_penalty_tx,
+    run_teosd,
+)
+
+common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="")
+
+# We'll use teos_cli to add appointments. The expected input format is a list of arguments with a json-encoded
+# appointment
+teos_cli.teos_api_server = "http://{}".format(HOST)
+teos_cli.teos_api_port = PORT
+
+# Run teosd
+teosd_process = run_teosd()
+
+
+def get_teos_pk():
+    teos_sk = Cryptographer.load_private_key_der(Cryptographer.load_key_file(config.get("TEOS_SECRET_KEY")))
+    teos_pk = teos_sk.public_key
+
+    return teos_pk
+
+
+def broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, addr):
+    # Broadcast the commitment transaction and mine a block
+    bitcoin_cli.sendrawtransaction(commitment_tx)
+    bitcoin_cli.generatetoaddress(1, addr)
+
+
+def get_appointment_info(locator):
+    # Check that the justice has been triggered (the appointment has moved from Watcher to Responder)
+    sleep(1)  # Let's add a bit of delay so the state can be updated
+    return teos_cli.get_appointment(locator)
+
+
+def test_appointment_life_cycle(monkeypatch, bitcoin_cli, create_txs):
+    monkeypatch.setattr(teos_cli, "load_keys", get_teos_pk)
+
+    commitment_tx, penalty_tx = create_txs
+    commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")
+    appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx)
+    locator = compute_locator(commitment_tx_id)
+
+    assert teos_cli.add_appointment([json.dumps(appointment_data)]) is True
+
+    appointment_info = get_appointment_info(locator)
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "being_watched"
+
+    new_addr = bitcoin_cli.getnewaddress()
+    broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr)
+
+    appointment_info = get_appointment_info(locator)
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "dispute_responded"
+
+    # It can be also checked by ensuring that the penalty transaction made it to the network
+    penalty_tx_id = bitcoin_cli.decoderawtransaction(penalty_tx).get("txid")
+
+    try:
+        bitcoin_cli.getrawtransaction(penalty_tx_id)
+        assert True
+
+    except JSONRPCException:
+        # If the transaction if not found.
+        assert False
+
+    # Now let's mine some blocks so the appointment reaches its end.
+    # Since we are running all the nodes remotely data may take more time than normal, and some confirmations may be
+    # missed, so we generate more than enough confirmations and add some delays.
+    for _ in range(int(1.5 * END_TIME_DELTA)):
+        sleep(1)
+        bitcoin_cli.generatetoaddress(1, new_addr)
+
+    appointment_info = get_appointment_info(locator)
+    assert appointment_info[0].get("status") == "not_found"
+
+
+def test_appointment_malformed_penalty(monkeypatch, bitcoin_cli, create_txs):
+    monkeypatch.setattr(teos_cli, "load_keys", get_teos_pk)
+
+    # Lets start by creating two valid transaction
+    commitment_tx, penalty_tx = create_txs
+
+    # Now we can modify the penalty so it is invalid when broadcast
+    mod_penalty_tx = Tx.from_hex(penalty_tx)
+    tx_in = mod_penalty_tx.tx_ins[0].copy(redeem_script=b"")
+    mod_penalty_tx = mod_penalty_tx.copy(tx_ins=[tx_in])
+
+    commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")
+    appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, mod_penalty_tx.hex())
+    locator = compute_locator(commitment_tx_id)
+
+    assert teos_cli.add_appointment([json.dumps(appointment_data)]) is True
+
+    # Broadcast the commitment transaction and mine a block
+    new_addr = bitcoin_cli.getnewaddress()
+    broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr)
+
+    # The appointment should have been removed since the penalty_tx was malformed.
+    sleep(1)
+    appointment_info = get_appointment_info(locator)
+
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "not_found"
+
+
+def test_appointment_wrong_key(bitcoin_cli, create_txs):
+    # This tests an appointment encrypted with a key that has not been derived from the same source as the locator.
+    # Therefore the tower won't be able to decrypt the blob once the appointment is triggered.
+    commitment_tx, penalty_tx = create_txs
+
+    # The appointment data is built using a random 32-byte value.
+    appointment_data = build_appointment_data(bitcoin_cli, get_random_value_hex(32), penalty_tx)
+
+    # We can't use teos_cli.add_appointment here since it computes the locator internally, so let's do it manually.
+    # We will encrypt the blob using the random value and derive the locator from the commitment tx.
+    appointment_data["locator"] = compute_locator(bitcoin_cli.decoderawtransaction(commitment_tx).get("txid"))
+    appointment_data["encrypted_blob"] = Cryptographer.encrypt(Blob(penalty_tx), get_random_value_hex(32))
+    appointment = Appointment.from_dict(appointment_data)
+
+    # teos_pk, cli_sk, cli_pk_der = teos_cli.load_keys(
+    #     cli_conf.get("TEOS_PUBLIC_KEY"), cli_conf.get("CLI_PRIVATE_KEY"), cli_conf.get("CLI_PUBLIC_KEY")
+    # )
+    # hex_pk_der = binascii.hexlify(cli_pk_der)
+    #
+    # signature = Cryptographer.sign(appointment.serialize(), cli_sk)
+    # data = {"appointment": appointment.to_dict(), "signature": signature, "public_key": hex_pk_der.decode("utf-8")}
+    # FIXME: Since the pk is now hardcoded for the alpha in the cli we cannot use load_keys here. We need to derive
+    #   the pk from the sk on disk.
+    teos_pk = get_teos_pk()
+    data = {"appointment": appointment.to_dict()}
+
+    # Send appointment to the server.
+    response = teos_cli.post_appointment(data)
+    response_json = teos_cli.process_post_appointment_response(response)
+
+    # Check that the server has accepted the appointment
+    signature = response_json.get("signature")
+    assert signature is not None
+    rpk = Cryptographer.recover_pk(appointment.serialize(), signature)
+    assert Cryptographer.verify_rpk(teos_pk, rpk) is True
+    assert response_json.get("locator") == appointment.locator
+
+    # Trigger the appointment
+    new_addr = bitcoin_cli.getnewaddress()
+    broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr)
+
+    # The appointment should have been removed since the decryption failed.
+    sleep(1)
+    appointment_info = get_appointment_info(appointment.locator)
+
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "not_found"
+
+
+def test_two_identical_appointments(monkeypatch, bitcoin_cli, create_txs):
+    monkeypatch.setattr(teos_cli, "load_keys", get_teos_pk)
+
+    # Tests sending two identical appointments to the tower.
+    # At the moment there are no checks for identical appointments, so both will be accepted, decrypted and kept until
+    # the end.
+    # TODO: 34-exact-duplicate-appointment
+    # This tests sending an appointment with two valid transaction with the same locator.
+    commitment_tx, penalty_tx = create_txs
+    commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")
+
+    appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx)
+    locator = compute_locator(commitment_tx_id)
+
+    # Send the appointment twice
+    assert teos_cli.add_appointment([json.dumps(appointment_data)]) is True
+    assert teos_cli.add_appointment([json.dumps(appointment_data)]) is True
+
+    # Broadcast the commitment transaction and mine a block
+    new_addr = bitcoin_cli.getnewaddress()
+    broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr)
+
+    # The first appointment should have made it to the Responder, and the second one should have been dropped for
+    # double-spending
+    sleep(1)
+    appointment_info = get_appointment_info(locator)
+
+    assert appointment_info is not None
+    assert len(appointment_info) == 2
+
+    for info in appointment_info:
+        assert info.get("status") == "dispute_responded"
+        assert info.get("penalty_rawtx") == penalty_tx
+
+
+def test_two_appointment_same_locator_different_penalty(monkeypatch, bitcoin_cli, create_txs):
+    monkeypatch.setattr(teos_cli, "load_keys", get_teos_pk)
+
+    # This tests sending an appointment with two valid transaction with the same locator.
+    commitment_tx, penalty_tx1 = create_txs
+    commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")
+
+    # We need to create a second penalty spending from the same commitment
+    decoded_commitment_tx = bitcoin_cli.decoderawtransaction(commitment_tx)
+    new_addr = bitcoin_cli.getnewaddress()
+    penalty_tx2 = create_penalty_tx(bitcoin_cli, decoded_commitment_tx, new_addr)
+
+    appointment1_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx1)
+    appointment2_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx2)
+    locator = compute_locator(commitment_tx_id)
+
+    assert teos_cli.add_appointment([json.dumps(appointment1_data)]) is True
+    assert teos_cli.add_appointment([json.dumps(appointment2_data)]) is True
+
+    # Broadcast the commitment transaction and mine a block
+    new_addr = bitcoin_cli.getnewaddress()
+    broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr)
+
+    # The first appointment should have made it to the Responder, and the second one should have been dropped for
+    # double-spending
+    sleep(1)
+    appointment_info = get_appointment_info(locator)
+
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "dispute_responded"
+    assert appointment_info[0].get("penalty_rawtx") == penalty_tx1
+
+
+def test_appointment_shutdown_teos_trigger_back_online(monkeypatch, create_txs, bitcoin_cli):
+    global teosd_process
+
+    monkeypatch.setattr(teos_cli, "load_keys", get_teos_pk)
+
+    teos_pid = teosd_process.pid
+
+    commitment_tx, penalty_tx = create_txs
+    commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")
+    appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx)
+    locator = compute_locator(commitment_tx_id)
+
+    assert teos_cli.add_appointment([json.dumps(appointment_data)]) is True
+
+    # Restart teos
+    teosd_process.terminate()
+    teosd_process = run_teosd()
+
+    assert teos_pid != teosd_process.pid
+
+    # Check that the appointment is still in the Watcher
+    appointment_info = get_appointment_info(locator)
+
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "being_watched"
+
+    # Trigger appointment after restart
+    new_addr = bitcoin_cli.getnewaddress()
+    broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr)
+
+    # The appointment should have been moved to the Responder
+    sleep(1)
+    appointment_info = get_appointment_info(locator)
+
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "dispute_responded"
+
+
+def test_appointment_shutdown_teos_trigger_while_offline(monkeypatch, create_txs, bitcoin_cli):
+    global teosd_process
+
+    monkeypatch.setattr(teos_cli, "load_keys", get_teos_pk)
+
+    teos_pid = teosd_process.pid
+
+    commitment_tx, penalty_tx = create_txs
+    commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")
+    appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx)
+    locator = compute_locator(commitment_tx_id)
+
+    assert teos_cli.add_appointment([json.dumps(appointment_data)]) is True
+
+    # Check that the appointment is still in the Watcher
+    appointment_info = get_appointment_info(locator)
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "being_watched"
+
+    # Shutdown and trigger
+    teosd_process.terminate()
+    new_addr = bitcoin_cli.getnewaddress()
+    broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr)
+
+    # Restart
+    teosd_process = run_teosd()
+    assert teos_pid != teosd_process.pid
+
+    # The appointment should have been moved to the Responder
+    sleep(1)
+    appointment_info = get_appointment_info(locator)
+
+    assert appointment_info is not None
+    assert len(appointment_info) == 1
+    assert appointment_info[0].get("status") == "dispute_responded"
+
+    teosd_process.terminate()