diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..74c5a28 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +omit = + pisa/pisad.py + pisa/sample_conf.py + pisa/time_traveler.py + pisa/utils/auth_proxy.py \ No newline at end of file diff --git a/pisa/__init__.py b/pisa/__init__.py index 5d03345..8aa5441 100644 --- a/pisa/__init__.py +++ b/pisa/__init__.py @@ -11,8 +11,3 @@ logging.basicConfig(format='%(message)s', level=logging.INFO, handlers=[ logging.FileHandler(conf.SERVER_LOG_FILE), logging.StreamHandler() ]) - -# Create RPC connection with bitcoind -# TODO: Check if a long lived connection like this may create problems (timeouts) -bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, - conf.BTC_RPC_PORT)) diff --git a/pisa/block_processor.py b/pisa/block_processor.py index a5f30e3..83b0441 100644 --- a/pisa/block_processor.py +++ b/pisa/block_processor.py @@ -1,8 +1,8 @@ import binascii from hashlib import sha256 -from pisa import bitcoin_cli from pisa.logger import Logger +from pisa.tools import bitcoin_cli from pisa.utils.auth_proxy import JSONRPCException logger = Logger("BlockProcessor") @@ -13,7 +13,7 @@ class BlockProcessor: def get_block(block_hash): try: - block = bitcoin_cli.getblock(block_hash) + block = bitcoin_cli().getblock(block_hash) except JSONRPCException as e: block = None @@ -25,7 +25,7 @@ class BlockProcessor: def get_best_block_hash(): try: - block_hash = bitcoin_cli.getbestblockhash() + block_hash = bitcoin_cli().getbestblockhash() except JSONRPCException as e: block_hash = None @@ -37,7 +37,7 @@ class BlockProcessor: def get_block_count(): try: - block_count = bitcoin_cli.getblockcount() + block_count = bitcoin_cli().getblockcount() except JSONRPCException as e: block_count = None @@ -73,7 +73,7 @@ class BlockProcessor: try: # ToDo: #20-test-tx-decrypting-edge-cases justice_rawtx = appointments[uuid].encrypted_blob.decrypt(dispute_txid) - justice_txid = bitcoin_cli.decoderawtransaction(justice_rawtx).get('txid') + justice_txid = bitcoin_cli().decoderawtransaction(justice_rawtx).get('txid') logger.info("Match found for locator.", locator=locator, uuid=uuid, justice_txid=justice_txid) except JSONRPCException as e: diff --git a/pisa/carrier.py b/pisa/carrier.py index d4cfbed..130cfca 100644 --- a/pisa/carrier.py +++ b/pisa/carrier.py @@ -1,6 +1,6 @@ from pisa.rpc_errors import * -from pisa import bitcoin_cli from pisa.logger import Logger +from pisa.tools import bitcoin_cli from pisa.utils.auth_proxy import JSONRPCException from pisa.errors import UNKNOWN_JSON_RPC_EXCEPTION @@ -20,7 +20,7 @@ class Carrier: def send_transaction(self, rawtx, txid): try: logger.info("Pushing transaction to the network", txid=txid, rawtx=rawtx) - bitcoin_cli.sendrawtransaction(rawtx) + bitcoin_cli().sendrawtransaction(rawtx) receipt = Receipt(delivered=True) @@ -70,7 +70,7 @@ class Carrier: @staticmethod def get_transaction(txid): try: - tx_info = bitcoin_cli.getrawtransaction(txid, 1) + tx_info = bitcoin_cli().getrawtransaction(txid, 1) except JSONRPCException as e: tx_info = None diff --git a/pisa/inspector.py b/pisa/inspector.py index 53055b8..1ae4547 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -2,7 +2,6 @@ import re from pisa import errors import pisa.conf as conf -from pisa import bitcoin_cli from pisa.logger import Logger from pisa.appointment import Appointment from pisa.block_processor import BlockProcessor diff --git a/pisa/responder.py b/pisa/responder.py index 6a1457d..dcce0a6 100644 --- a/pisa/responder.py +++ b/pisa/responder.py @@ -1,6 +1,7 @@ +import json from queue import Queue -from threading import Thread from hashlib import sha256 +from threading import Thread from binascii import unhexlify from pisa.logger import Logger @@ -34,6 +35,9 @@ class Job: return job + def to_json(self): + return json.dumps(self.to_dict()) + class Responder: def __init__(self): @@ -62,6 +66,8 @@ class Responder: # TODO: Add the missing reasons (e.g. RPC_VERIFY_REJECTED) pass + return receipt + def create_job(self, uuid, dispute_txid, justice_txid, justice_rawtx, appointment_end, confirmations=0, retry=False): @@ -155,7 +161,7 @@ class Responder: for uuid, job in self.jobs.items(): if job.appointment_end <= height: - tx = Carrier.get_transaction(job.dispute_txid) + tx = Carrier.get_transaction(job.justice_txid) # FIXME: Should be improved with the librarian confirmations = tx.get('confirmations') diff --git a/pisa/tools.py b/pisa/tools.py index aeaa310..adba5e5 100644 --- a/pisa/tools.py +++ b/pisa/tools.py @@ -1,10 +1,15 @@ import re from http.client import HTTPException -from pisa import bitcoin_cli +import pisa.conf as conf from pisa.logger import Logger -from pisa.utils.auth_proxy import JSONRPCException from pisa.rpc_errors import RPC_INVALID_ADDRESS_OR_KEY +from pisa.utils.auth_proxy import AuthServiceProxy, JSONRPCException + + +def bitcoin_cli(): + return AuthServiceProxy("http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, + conf.BTC_RPC_PORT)) # TODO: currently only used in the Responder; might move there or in the BlockProcessor @@ -13,7 +18,7 @@ def check_tx_in_chain(tx_id, logger=Logger(), tx_label='Transaction'): confirmations = 0 try: - tx_info = bitcoin_cli.getrawtransaction(tx_id, 1) + tx_info = bitcoin_cli().getrawtransaction(tx_id, 1) if tx_info.get("confirmations"): confirmations = int(tx_info.get("confirmations")) @@ -38,7 +43,7 @@ def can_connect_to_bitcoind(): can_connect = True try: - bitcoin_cli.help() + bitcoin_cli().help() except (ConnectionRefusedError, JSONRPCException, HTTPException): can_connect = False @@ -50,7 +55,7 @@ def in_correct_network(network): testnet3_genesis_block_hash = "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943" correct_network = False - genesis_block_hash = bitcoin_cli.getblockhash(0) + genesis_block_hash = bitcoin_cli().getblockhash(0) if network == 'mainnet' and genesis_block_hash == mainnet_genesis_block_hash: correct_network = True @@ -65,3 +70,4 @@ def in_correct_network(network): def check_txid_format(txid): # TODO: #12-check-txid-regexp return isinstance(txid, str) and re.search(r'^[0-9A-Fa-f]{64}$', txid) is not None + diff --git a/test/simulator/bitcoin_sim_tests.py b/test/simulator/bitcoin_sim_tests.py index cb1c5bb..f0eafe5 100644 --- a/test/simulator/bitcoin_sim_tests.py +++ b/test/simulator/bitcoin_sim_tests.py @@ -1,107 +1,146 @@ +import re import os +import pytest +from time import sleep +from threading import Thread + +from test.simulator.transaction import TX +from test.simulator.bitcoind_sim import run_simulator from pisa.utils.auth_proxy import AuthServiceProxy, JSONRPCException from pisa.conf import BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT -from pisa.tools import check_txid_format + +MIXED_VALUES = values = [-1, 500, '', '111', [], 1.1, None, '', "a" * 31, "b" * 33, os.urandom(32).hex()] + + +@pytest.fixture(scope='module') +def run_bitcoind(): + bitcoind_thread = Thread(target=run_simulator, kwargs={"mode": "event"}) + bitcoind_thread.daemon = True + bitcoind_thread.start() + + # It takes a little bit of time to start the API (otherwise the requests are sent too early and they fail) + sleep(0.1) + + +@pytest.fixture(scope="module") +def genesis_block_hash(run_bitcoind): + return bitcoin_cli.getblockhash(0) + + +def check_hash_format(txid): + # TODO: #12-check-txid-regexp + return isinstance(txid, str) and re.search(r'^[0-9A-Fa-f]{64}$', txid) is not None bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT)) -# Help should always return 0 -assert(bitcoin_cli.help() == 0) -# getblockhash should return a blockid (which matches the txid format) -block_hash = bitcoin_cli.getblockhash(0) -assert(check_txid_format(block_hash)) +def test_help(run_bitcoind): + # Help should always return 0 + assert(bitcoin_cli.help() == 0) -# Check that the values are within range and of the proper format (all should fail) -values = [-1, 500, None, '', '111', [], 1.1] -print("getblockhash fails ({}):".format(len(values))) -for v in values: +# FIXME: Better assert for the exceptions would be nice (check the returned errno is the expected one) + +def test_getblockhash(genesis_block_hash): + # First block + assert(check_hash_format(genesis_block_hash)) + + # Check that the values are within range and of the proper format (all should fail) + for v in MIXED_VALUES: + try: + bitcoin_cli.getblockhash(v) + assert False + except JSONRPCException as e: + assert True + + +def test_get_block(genesis_block_hash): + # getblock should return a list of transactions and the height + block = bitcoin_cli.getblock(genesis_block_hash) + assert(isinstance(block.get('tx'), list)) + assert(len(block.get('tx')) != 0) + assert(isinstance(block.get('height'), int)) + + # It should fail for wrong data formats and random ids + for v in MIXED_VALUES: + try: + bitcoin_cli.getblock(v) + assert False + except JSONRPCException as e: + assert True + + +def test_decoderawtransaction(genesis_block_hash): + # decoderawtransaction should only return if the given transaction matches a txid format + block = bitcoin_cli.getblock(genesis_block_hash) + coinbase_txid = block.get('tx')[0] + + coinbase_tx = bitcoin_cli.getrawtransaction(coinbase_txid).get("hex") + tx = bitcoin_cli.decoderawtransaction(coinbase_tx) + + assert(isinstance(tx, dict)) + assert(isinstance(tx.get('txid'), str)) + assert(check_hash_format(tx.get('txid'))) + + # Therefore should also work for a random transaction hex in our simulation + random_tx = TX.create_dummy_transaction() + tx = bitcoin_cli.decoderawtransaction(random_tx) + assert(isinstance(tx, dict)) + assert(isinstance(tx.get('txid'), str)) + assert(check_hash_format(tx.get('txid'))) + + # But it should fail for not proper formatted one + for v in MIXED_VALUES: + try: + bitcoin_cli.decoderawtransaction(v) + assert False + except JSONRPCException as e: + assert True + + +def test_sendrawtransaction(genesis_block_hash): + # sendrawtransaction should only allow txids that the simulator has not mined yet + bitcoin_cli.sendrawtransaction(TX.create_dummy_transaction()) + + # Any data not matching the txid format or that matches with an already mined transaction should fail try: - block_hash = bitcoin_cli.getblockhash(v) + genesis_tx = bitcoin_cli.getblock(genesis_block_hash).get("tx")[0] + bitcoin_cli.sendrawtransaction(genesis_tx) assert False + except JSONRPCException as e: - print('\t{}'.format(e)) + assert True -# getblock should return a list of transactions and the height -block = bitcoin_cli.getblock(block_hash) -assert(isinstance(block.get('tx'), list)) -assert(len(block.get('tx')) != 0) -assert(isinstance(block.get('height'), int)) + for v in MIXED_VALUES: + try: + bitcoin_cli.sendrawtransaction(v) + assert False + except JSONRPCException as e: + assert True -# Some fails -values += ["a"*64, os.urandom(32).hex()] -print("\ngetblock fails ({}):".format(len(values))) -for v in values: - try: - block = bitcoin_cli.getblock(v) - assert False - except JSONRPCException as e: - print('\t{}'.format(e)) +def test_getrawtransaction(genesis_block_hash): + # getrawtransaction should work for existing transactions, and fail for non-existing ones + genesis_tx = bitcoin_cli.getblock(genesis_block_hash).get("tx")[0] + tx = bitcoin_cli.getrawtransaction(genesis_tx) -# decoderawtransaction should only return if the given transaction matches a txid format -coinbase_tx = block.get('tx')[0] -tx = bitcoin_cli.decoderawtransaction(coinbase_tx) -assert(isinstance(tx, dict)) -assert(isinstance(tx.get('txid'), str)) -assert(check_txid_format(tx.get('txid'))) + assert(isinstance(tx, dict)) + assert(isinstance(tx.get('confirmations'), int)) -# Therefore should also work for a random formatted 32-byte hex in our simulation -random_tx = os.urandom(32).hex() -tx = bitcoin_cli.decoderawtransaction(random_tx) -assert(isinstance(tx, dict)) -assert(isinstance(tx.get('txid'), str)) -assert(check_txid_format(tx.get('txid'))) + for v in MIXED_VALUES: + try: + bitcoin_cli.getrawtransaction(v) + assert False + except JSONRPCException as e: + assert True -# But it should fail for not proper formatted one -values = [1, None, '', "a"*63, "b"*65, [], os.urandom(31).hex()] -print("\ndecoderawtransaction fails ({}):".format(len(values))) -for v in values: - try: - block = bitcoin_cli.decoderawtransaction(v) - assert False - except JSONRPCException as e: - print('\t{}'.format(e)) - -# sendrawtransaction should only allow txids that the simulator has not mined yet -bitcoin_cli.sendrawtransaction(os.urandom(32).hex()) - -# Any data not matching the txid format or that matches with an already mined transaction should fail -values += [coinbase_tx] - -print("\nsendrawtransaction fails ({}):".format(len(values))) - -for v in values: - try: - block = bitcoin_cli.sendrawtransaction(v) - assert False - except JSONRPCException as e: - print('\t{}'.format(e)) - -# getrawtransaction should work for existing transactions, and fail for non-existing ones -tx = bitcoin_cli.getrawtransaction(coinbase_tx) - -assert(isinstance(tx, dict)) -assert(isinstance(tx.get('confirmations'), int)) - -print("\nsendrawtransaction fails ({}):".format(len(values))) - -for v in values: - try: - block = bitcoin_cli.sendrawtransaction(v) - assert False - except JSONRPCException as e: - print('\t{}'.format(e)) - -# getblockcount should always return a positive integer -bc = bitcoin_cli.getblockcount() -assert (isinstance(bc, int)) -assert (bc >= 0) - -print("\nAll tests passed!") +def test_getblockcount(): + # getblockcount should always return a positive integer + bc = bitcoin_cli.getblockcount() + assert (isinstance(bc, int)) + assert (bc >= 0) diff --git a/test/simulator/bitcoind_sim.py b/test/simulator/bitcoind_sim.py index b481b43..4306f13 100644 --- a/test/simulator/bitcoind_sim.py +++ b/test/simulator/bitcoind_sim.py @@ -1,26 +1,36 @@ -from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT -from flask import Flask, request, Response, abort -from test.simulator.zmq_publisher import ZMQPublisher -from threading import Thread -from pisa.rpc_errors import * -from pisa.tools import check_txid_format -import logging -import binascii -import json import os import time +import json +import logging +import binascii +from threading import Thread, Event +from flask import Flask, request, Response, abort +from pisa.rpc_errors import * +from test.simulator.utils import sha256d +from test.simulator.transaction import TX +from test.simulator.zmq_publisher import ZMQPublisher +from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT app = Flask(__name__) HOST = 'localhost' PORT = '18443' -TIME_BETWEEN_BLOCKS = 10 - -mempool = [] -mined_transactions = {} -blocks = {} blockchain = [] +blocks = {} +mined_transactions = {} +mempool = [] + +mine_new_block = Event() + + +@app.route('/generate', methods=['POST']) +def generate(): + global mine_new_block + + mine_new_block.set() + + return Response(status=200, mimetype='application/json') @app.route('/', methods=['POST']) @@ -67,10 +77,12 @@ def process_request(): no_param_err = {"code": RPC_MISC_ERROR, "message": "JSON value is not a {} as expected"} if method == "decoderawtransaction": - txid = get_param(request_data) + rawtx = get_param(request_data) - if isinstance(txid, str): - if check_txid_format(txid): + if isinstance(rawtx, str) and len(rawtx) % 2 is 0: + txid = sha256d(rawtx) + + if TX.deserialize(rawtx) is not None: response["result"] = {"txid": txid} else: @@ -82,12 +94,15 @@ def process_request(): elif method == "sendrawtransaction": # TODO: A way of rejecting transactions should be added to test edge cases. - txid = get_param(request_data) + rawtx = get_param(request_data) - if isinstance(txid, str): - if check_txid_format(txid): + if isinstance(rawtx, str) and len(rawtx) % 2 is 0: + txid = sha256d(rawtx) + + if TX.deserialize(rawtx) is not None: if txid not in list(mined_transactions.keys()): - mempool.append(txid) + mempool.append(rawtx) + response["result"] = {"txid": txid} else: response["error"] = {"code": RPC_VERIFY_ALREADY_IN_CHAIN, @@ -104,10 +119,10 @@ def process_request(): txid = get_param(request_data) if isinstance(txid, str): - block = blocks.get(mined_transactions.get(txid)) - - if block: - response["result"] = {"confirmations": len(blockchain) - block.get('height')} + if txid in mined_transactions: + block = blocks.get(mined_transactions[txid]["block"]) + rawtx = mined_transactions[txid].get('tx') + response["result"] = {"hex": rawtx, "confirmations": len(blockchain) - block.get('height')} elif txid in mempool: response["result"] = {"confirmations": 0} @@ -169,6 +184,7 @@ def get_param(request_data): param = None params = request_data.get("params") + if isinstance(params, list) and len(params) > 0: param = params[0] @@ -179,40 +195,57 @@ def load_data(): pass -def simulate_mining(): - global mempool, mined_transactions, blocks, blockchain +def simulate_mining(mode, time_between_blocks): + global mempool, mined_transactions, blocks, blockchain, mine_new_block prev_block_hash = None mining_simulator = ZMQPublisher(topic=b'hashblock', feed_protocol=FEED_PROTOCOL, feed_addr=FEED_ADDR, feed_port=FEED_PORT) - while True: + # Set the mining event to initialize the blockchain with a block + mine_new_block.set() + + while mine_new_block.wait(): block_hash = os.urandom(32).hex() - coinbase_tx_hash = os.urandom(32).hex() - txs_to_mine = [coinbase_tx_hash] + coinbase_tx = TX.create_dummy_transaction() + coinbase_tx_hash = sha256d(coinbase_tx) + + txs_to_mine = dict({coinbase_tx_hash: coinbase_tx}) if len(mempool) != 0: # We'll mine up to 100 txs per block - txs_to_mine += mempool[:99] + for rawtx in mempool[:99]: + txid = sha256d(rawtx) + txs_to_mine[txid] = rawtx + mempool = mempool[99:] # Keep track of the mined transaction (to respond to getrawtransaction) - for tx in txs_to_mine: - mined_transactions[tx] = block_hash + for txid, tx in txs_to_mine.items(): + mined_transactions[txid] = {"tx": tx, "block": block_hash} + + blocks[block_hash] = {"tx": list(txs_to_mine.keys()), "height": len(blockchain), + "previousblockhash": prev_block_hash} - blocks[block_hash] = {"tx": txs_to_mine, "height": len(blockchain), "previousblockhash": prev_block_hash} mining_simulator.publish_data(binascii.unhexlify(block_hash)) blockchain.append(block_hash) prev_block_hash = block_hash print("New block mined: {}".format(block_hash)) - print("\tTransactions: {}".format(txs_to_mine)) + print("\tTransactions: {}".format(list(txs_to_mine.keys()))) - time.sleep(TIME_BETWEEN_BLOCKS) + if mode == 'time': + time.sleep(time_between_blocks) + else: + mine_new_block.clear() + -def run_simulator(): - mining_thread = Thread(target=simulate_mining) +def run_simulator(mode='time', time_between_blocks=5): + if mode not in ["time", 'event']: + raise ValueError("Node must be time or event") + + mining_thread = Thread(target=simulate_mining, args=[mode, time_between_blocks]) mining_thread.start() # Setting Flask log to ERROR only so it does not mess with out logging diff --git a/test/simulator/transaction.py b/test/simulator/transaction.py new file mode 100644 index 0000000..be49a79 --- /dev/null +++ b/test/simulator/transaction.py @@ -0,0 +1,150 @@ +# Porting some functionality from https://github.com/sr-gi/bitcoin_tools with some modifications <3 +from os import urandom + +from test.simulator.utils import * + + +class TX: + """ Defines a class TX (transaction) that holds all the modifiable fields of a Bitcoin transaction, such as + version, number of inputs, reference to previous transactions, input and output scripts, value, etc. + """ + + def __init__(self): + self.version = None + self.inputs = None + self.outputs = None + self.nLockTime = None + self.prev_tx_id = [] + self.prev_out_index = [] + self.scriptSig = [] + self.scriptSig_len = [] + self.nSequence = [] + self.value = [] + self.scriptPubKey = [] + self.scriptPubKey_len = [] + + self.offset = 0 + self.hex = "" + + @classmethod + def deserialize(cls, hex_tx): + """ Builds a transaction object from the hexadecimal serialization format of a transaction that + could be obtained, for example, from a blockexplorer. + :param hex_tx: Hexadecimal serialized transaction. + :type hex_tx: hex str + :return: The transaction build using the provided hex serialized transaction. + :rtype: TX + """ + + tx = cls() + tx.hex = hex_tx + + try: + tx.version = int(change_endianness(parse_element(tx, 4)), 16) + + # INPUTS + tx.inputs = int(parse_varint(tx), 16) + + for i in range(tx.inputs): + tx.prev_tx_id.append(change_endianness(parse_element(tx, 32))) + tx.prev_out_index.append(int(change_endianness(parse_element(tx, 4)), 16)) + # ScriptSig + tx.scriptSig_len.append(int(parse_varint(tx), 16)) + tx.scriptSig.append(parse_element(tx, tx.scriptSig_len[i])) + tx.nSequence.append(int(parse_element(tx, 4), 16)) + + # OUTPUTS + tx.outputs = int(parse_varint(tx), 16) + + for i in range(tx.outputs): + tx.value.append(int(change_endianness(parse_element(tx, 8)), 16)) + # ScriptPubKey + tx.scriptPubKey_len.append(int(parse_varint(tx), 16)) + tx.scriptPubKey.append(parse_element(tx, tx.scriptPubKey_len[i])) + + tx.nLockTime = int(parse_element(tx, 4), 16) + + if tx.offset != len(tx.hex): + # There is some error in the serialized transaction passed as input. Transaction can't be built + tx = None + else: + tx.offset = 0 + + except ValueError: + # If a parsing error occurs, the deserialization stops and None is returned + tx = None + + return tx + + def serialize(self, rtype=hex): + """ Serialize all the transaction fields arranged in the proper order, resulting in a hexadecimal string + ready to be broadcast to the network. + :param self: self + :type self: TX + :param rtype: Whether the serialized transaction is returned as a hex str or a byte array. + :type rtype: hex or bool + :return: Serialized transaction representation (hexadecimal or bin depending on rtype parameter). + :rtype: hex str / bin + """ + + if rtype not in [hex, bin]: + raise Exception("Invalid return type (rtype). It should be either hex or bin.") + serialized_tx = change_endianness(int2bytes(self.version, 4)) # 4-byte version number (LE). + + # INPUTS + serialized_tx += encode_varint(self.inputs) # Varint number of inputs. + + for i in range(self.inputs): + serialized_tx += change_endianness(self.prev_tx_id[i]) # 32-byte hash of the previous transaction (LE). + serialized_tx += change_endianness(int2bytes(self.prev_out_index[i], 4)) # 4-byte output index (LE) + serialized_tx += encode_varint(len(self.scriptSig[i]) // 2) # Varint input script length. + # ScriptSig + serialized_tx += self.scriptSig[i] # Input script. + serialized_tx += int2bytes(self.nSequence[i], 4) # 4-byte sequence number. + + # OUTPUTS + serialized_tx += encode_varint(self.outputs) # Varint number of outputs. + + if self.outputs != 0: + for i in range(self.outputs): + serialized_tx += change_endianness(int2bytes(self.value[i], 8)) # 8-byte field Satoshi value (LE) + # ScriptPubKey + serialized_tx += encode_varint(len(self.scriptPubKey[i]) // 2) # Varint Output script length. + serialized_tx += self.scriptPubKey[i] # Output script. + + serialized_tx += int2bytes(self.nLockTime, 4) # 4-byte lock time field + + # If return type has been set to binary, the serialized transaction is converted. + if rtype is bin: + serialized_tx = unhexlify(serialized_tx) + + return serialized_tx + + @staticmethod + def create_dummy_transaction(prev_tx_id=None, prev_out_index=None): + tx = TX() + + if prev_tx_id is None: + prev_tx_id = urandom(32).hex() + + if prev_out_index is None: + prev_out_index = 0 + + tx.version = 1 + tx.inputs = 1 + tx.outputs = 1 + tx.prev_tx_id = [prev_tx_id] + tx.prev_out_index = [prev_out_index] + tx.nLockTime = 0 + tx.scriptSig = [ + '47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860' + 'a4acdd12909d831cc56cbbac4622082221a8768d1d0901'] + tx.scriptSig_len = [77] + tx.nSequence = [4294967295] + tx.value = [5000000000] + tx.scriptPubKey = [ + '4104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c' + '1b7303b8a0626f1baded5c72a704f7e6cd84cac'] + tx.scriptPubKey_len = [67] + + return tx.serialize() diff --git a/test/simulator/utils.py b/test/simulator/utils.py new file mode 100644 index 0000000..f2f2883 --- /dev/null +++ b/test/simulator/utils.py @@ -0,0 +1,128 @@ +# Porting some functionality from https://github.com/sr-gi/bitcoin_tools with some modifications <3 +from hashlib import sha256 +from binascii import unhexlify + + +def change_endianness(x): + """ Changes the endianness (from BE to LE and vice versa) of a given value. + :param x: Given value which endianness will be changed. + :type x: hex str + :return: The opposite endianness representation of the given value. + :rtype: hex str + """ + + # If there is an odd number of elements, we make it even by adding a 0 + if (len(x) % 2) == 1: + x += "0" + + y = bytes(x, 'utf-8') + z = y[::-1] + return z.decode('utf-8') + + +def parse_varint(tx): + """ Parses a given transaction for extracting an encoded varint element. + :param tx: Transaction where the element will be extracted. + :type tx: TX + :return: The b-bytes representation of the given value (a) in hex format. + :rtype: hex str + """ + + # First of all, the offset of the hex transaction if moved to the proper position (i.e where the varint should be + # located) and the length and format of the data to be analyzed is checked. + data = tx.hex[tx.offset:] + assert (len(data) > 0) + size = int(data[:2], 16) + assert (size <= 255) + + # Then, the integer is encoded as a varint using the proper prefix, if needed. + if size <= 252: # No prefix + storage_length = 1 + elif size == 253: # 0xFD + storage_length = 3 + elif size == 254: # 0xFE + storage_length = 5 + elif size == 255: # 0xFF + storage_length = 9 + else: + raise Exception("Wrong input data size") + + # Finally, the storage length is used to extract the proper number of bytes from the transaction hex and the + # transaction offset is updated. + varint = data[:storage_length * 2] + tx.offset += storage_length * 2 + + return varint + + +def parse_element(tx, size): + """ Parses a given transaction to extract an element of a given size. + :param tx: Transaction where the element will be extracted. + :type tx: TX + :param size: Size of the parameter to be extracted. + :type size: int + :return: The extracted element. + :rtype: hex str + """ + + element = tx.hex[tx.offset:tx.offset + size * 2] + tx.offset += size * 2 + return element + + +def encode_varint(value): + """ Encodes a given integer value to a varint. It only used the four varint representation cases used by bitcoin: + 1-byte, 2-byte, 4-byte or 8-byte integers. + :param value: The integer value that will be encoded into varint. + :type value: int + :return: The varint representation of the given integer value. + :rtype: str + """ + + # The value is checked in order to choose the size of its final representation. + # 0xFD(253), 0xFE(254) and 0xFF(255) are special cases, since are the prefixes defined for 2-byte, 4-byte + # and 8-byte long values respectively. + if value < pow(2, 8) - 3: + size = 1 + varint = int2bytes(value, size) # No prefix + else: + if value < pow(2, 16): + size = 2 + prefix = 253 # 0xFD + elif value < pow(2, 32): + size = 4 + prefix = 254 # 0xFE + elif value < pow(2, 64): + size = 8 + prefix = 255 # 0xFF + else: + raise Exception("Wrong input data size") + varint = format(prefix, 'x') + change_endianness(int2bytes(value, size)) + + return varint + + +def int2bytes(a, b): + """ Converts a given integer value (a) its b-byte representation, in hex format. + :param a: Value to be converted. + :type a: int + :param b: Byte size to be filled. + :type b: int + :return: The b-bytes representation of the given value (a) in hex format. + :rtype: hex str + """ + + m = pow(2, 8*b) - 1 + if a > m: + raise Exception(str(a) + " is too big to be represented with " + str(b) + " bytes. Maximum value is " + + str(m) + ".") + + return ('%0' + str(2 * b) + 'x') % a + + +def sha256d(hex_data): + data = unhexlify(hex_data) + double_sha256 = sha256(sha256(data).digest()).hexdigest() + + return change_endianness(double_sha256) + diff --git a/test/unit/conftest.py b/test/unit/conftest.py index ab6af2b..2c0bdea 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -1,14 +1,15 @@ import pytest +import requests from time import sleep from threading import Thread from pisa.api import start_api -from test.simulator.bitcoind_sim import run_simulator +from test.simulator.bitcoind_sim import run_simulator, HOST, PORT @pytest.fixture(scope='session') def run_bitcoind(): - bitcoind_thread = Thread(target=run_simulator) + bitcoind_thread = Thread(target=run_simulator, kwargs={"mode": "event"}) bitcoind_thread.daemon = True bitcoind_thread.start() @@ -24,3 +25,10 @@ def run_api(): # It takes a little bit of time to start the API (otherwise the requests are sent too early and they fail) sleep(0.1) + + +def generate_block(): + requests.post(url="http://{}:{}/generate".format(HOST, PORT), timeout=5) + sleep(0.5) + + diff --git a/test/unit/test_api.py b/test/unit/test_api.py index 8a5661d..ba052e5 100644 --- a/test/unit/test_api.py +++ b/test/unit/test_api.py @@ -1,37 +1,42 @@ -import os import json import pytest -import time import requests from hashlib import sha256 from binascii import unhexlify from apps.cli.blob import Blob from pisa import HOST, PORT, logging +from test.simulator.utils import sha256d +from test.simulator.transaction import TX +from test.unit.conftest import generate_block from pisa.utils.auth_proxy import AuthServiceProxy -from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS from pisa.conf import BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT, MAX_APPOINTMENTS logging.getLogger().disabled = True + PISA_API = "http://{}:{}".format(HOST, PORT) MULTIPLE_APPOINTMENTS = 10 appointments = [] -locator_dispute_txid_map = {} +locator_dispute_tx_map = {} -def generate_dummy_appointment(dispute_txid): +def generate_dummy_appointment(): r = requests.get(url=PISA_API + '/get_block_count', timeout=5) current_height = r.json().get("block_count") - dummy_appointment_data = {"tx": os.urandom(32).hex(), "tx_id": dispute_txid, "start_time": current_height + 5, + dispute_tx = TX.create_dummy_transaction() + dispute_txid = sha256d(dispute_tx) + justice_tx = TX.create_dummy_transaction(dispute_txid) + + dummy_appointment_data = {"tx": justice_tx, "tx_id": dispute_txid, "start_time": current_height + 5, "end_time": current_height + 30, "dispute_delta": 20} cipher = "AES-GCM-128" hash_function = "SHA256" - locator = sha256(unhexlify(dummy_appointment_data.get("tx_id"))).hexdigest() + locator = sha256(unhexlify(dispute_txid)).hexdigest() blob = Blob(dummy_appointment_data.get("tx"), cipher, hash_function) encrypted_blob = blob.encrypt((dummy_appointment_data.get("tx_id"))) @@ -41,22 +46,13 @@ def generate_dummy_appointment(dispute_txid): "dispute_delta": dummy_appointment_data.get("dispute_delta"), "encrypted_blob": encrypted_blob, "cipher": cipher, "hash_function": hash_function} - return appointment + return appointment, dispute_tx @pytest.fixture -def new_appointment(dispute_txid=None): - appointment = create_appointment(dispute_txid) - - return appointment - - -def create_appointment(dispute_txid=None): - if dispute_txid is None: - dispute_txid = os.urandom(32).hex() - - appointment = generate_dummy_appointment(dispute_txid) - locator_dispute_txid_map[appointment["locator"]] = dispute_txid +def new_appointment(): + appointment, dispute_tx = generate_dummy_appointment() + locator_dispute_tx_map[appointment["locator"]] = dispute_tx return appointment @@ -147,12 +143,12 @@ def test_get_all_appointments_responder(): bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT)) locators = [appointment["locator"] for appointment in appointments] - for locator, dispute_txid in locator_dispute_txid_map.items(): + for locator, dispute_tx in locator_dispute_tx_map.items(): if locator in locators: - bitcoin_cli.sendrawtransaction(dispute_txid) + bitcoin_cli.sendrawtransaction(dispute_tx) # Wait a bit for them to get confirmed - time.sleep(TIME_BETWEEN_BLOCKS) + generate_block() # Get all appointments r = requests.get(url=PISA_API + "/get_all_appointments") diff --git a/test/unit/test_carrier.py b/test/unit/test_carrier.py index 374dd24..d73c749 100644 --- a/test/unit/test_carrier.py +++ b/test/unit/test_carrier.py @@ -1,11 +1,13 @@ import pytest import logging from os import urandom -from time import sleep + from pisa.carrier import Carrier +from test.simulator.utils import sha256d +from test.simulator.transaction import TX +from test.unit.conftest import generate_block from pisa.rpc_errors import RPC_VERIFY_ALREADY_IN_CHAIN, RPC_DESERIALIZATION_ERROR -from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS logging.getLogger().disabled = True @@ -24,23 +26,28 @@ def carrier(): def test_send_transaction(run_bitcoind, carrier): # We are mocking bitcoind and in our simulator txid == tx - tx = urandom(32).hex() - receipt = carrier.send_transaction(tx, tx) + tx = TX.create_dummy_transaction() + txid = sha256d(tx) + + receipt = carrier.send_transaction(tx, txid) assert(receipt.delivered is True) def test_send_double_spending_transaction(carrier): # We can test what happens if the same transaction is sent twice - tx = urandom(32).hex() - receipt = carrier.send_transaction(tx, tx) - sent_txs.append(tx) + tx = TX.create_dummy_transaction() + txid = sha256d(tx) + + receipt = carrier.send_transaction(tx, txid) + sent_txs.append(txid) # Wait for a block to be mined - sleep(2*TIME_BETWEEN_BLOCKS) + for _ in range(2): + generate_block() # Try to send it again - receipt2 = carrier.send_transaction(tx, tx) + receipt2 = carrier.send_transaction(tx, txid) # The carrier should report delivered True for both, but in the second case the transaction was already delivered # (either by himself or someone else) @@ -51,8 +58,9 @@ def test_send_double_spending_transaction(carrier): def test_send_transaction_invalid_format(carrier): # Test sending a transaction that does not fits the format - tx = urandom(31).hex() - receipt = carrier.send_transaction(tx, tx) + tx = TX.create_dummy_transaction() + txid = sha256d(tx) + receipt = carrier.send_transaction(txid, txid) assert (receipt.delivered is False and receipt.reason == RPC_DESERIALIZATION_ERROR) diff --git a/test/unit/test_encrypted_blob.py b/test/unit/test_encrypted_blob.py index 26e1d9e..25dc78f 100644 --- a/test/unit/test_encrypted_blob.py +++ b/test/unit/test_encrypted_blob.py @@ -1,5 +1,4 @@ from os import urandom -from cryptography.exceptions import InvalidTag from pisa import logging from pisa.encrypted_blob import EncryptedBlob diff --git a/test/unit/test_tools.py b/test/unit/test_tools.py index e3bce92..251663e 100644 --- a/test/unit/test_tools.py +++ b/test/unit/test_tools.py @@ -1,4 +1,4 @@ -from pisa import logging, bitcoin_cli +from pisa import logging from pisa.tools import check_txid_format from pisa.tools import can_connect_to_bitcoind, in_correct_network diff --git a/test/unit/test_watcher.py b/test/unit/test_watcher.py index 6d5476b..db19815 100644 --- a/test/unit/test_watcher.py +++ b/test/unit/test_watcher.py @@ -1,23 +1,28 @@ import pytest import logging -from os import urandom -from time import sleep from uuid import uuid4 from hashlib import sha256 from threading import Thread +from binascii import unhexlify from queue import Queue, Empty +from apps.cli.blob import Blob from pisa.watcher import Watcher from pisa.responder import Responder from pisa.conf import MAX_APPOINTMENTS from pisa.appointment import Appointment from pisa.tools import check_txid_format +from test.simulator.utils import sha256d +from test.simulator.transaction import TX +from test.unit.conftest import generate_block from pisa.utils.auth_proxy import AuthServiceProxy -from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS from pisa.conf import EXPIRY_DELTA, BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT logging.getLogger().disabled = True + APPOINTMENTS = 5 +START_TIME_OFFSET = 1 +END_TIME_OFFSET = 1 @pytest.fixture(scope="module") @@ -25,37 +30,44 @@ def watcher(): return Watcher() -def create_appointment(locator=None): +def generate_dummy_appointment(): bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT)) - if locator is None: - locator = urandom(32).hex() + dispute_tx = TX.create_dummy_transaction() + dispute_txid = sha256d(dispute_tx) + justice_tx = TX.create_dummy_transaction(dispute_txid) start_time = bitcoin_cli.getblockcount() + 1 end_time = start_time + 1 dispute_delta = 20 - encrypted_blob_data = urandom(100).hex() + cipher = "AES-GCM-128" hash_function = "SHA256" - return Appointment(locator, start_time, end_time, dispute_delta, encrypted_blob_data, cipher, hash_function) + locator = sha256(unhexlify(dispute_txid)).hexdigest() + blob = Blob(justice_tx, cipher, hash_function) + + encrypted_blob = blob.encrypt(dispute_txid) + + appointment = Appointment(locator, start_time, end_time, dispute_delta, encrypted_blob, cipher, hash_function) + + return appointment, dispute_tx def create_appointments(n): locator_uuid_map = dict() appointments = dict() - txids = [] + dispute_txs = [] for i in range(n): - txid = urandom(32) + appointment, dispute_tx = generate_dummy_appointment() uuid = uuid4().hex - locator = sha256(txid).hexdigest() - appointments[uuid] = create_appointment(locator) - locator_uuid_map[locator] = [uuid] - txids.append(txid.hex()) + appointments[uuid] = appointment + locator_uuid_map[appointment.locator] = [uuid] + dispute_txs.append(dispute_tx) - return appointments, locator_uuid_map, txids + return appointments, locator_uuid_map, dispute_txs def test_init(watcher): @@ -69,13 +81,14 @@ def test_init(watcher): def test_add_appointment(run_bitcoind, watcher): - # The watcher automatically fire do_watch and do_subscribe on adding an appointment if it is asleep (initial state). + # The watcher automatically fires do_watch and do_subscribe on adding an appointment if it is asleep (initial state) # Avoid this by setting the state to awake. watcher.asleep = False # We should be able to add appointments up to the limit for _ in range(10): - added_appointment, sig = watcher.add_appointment(create_appointment()) + appointment, dispute_tx = generate_dummy_appointment() + added_appointment, sig = watcher.add_appointment(appointment) assert added_appointment is True @@ -85,11 +98,13 @@ def test_add_too_many_appointments(watcher): watcher.appointments = dict() for _ in range(MAX_APPOINTMENTS): - added_appointment, sig = watcher.add_appointment(create_appointment()) + appointment, dispute_tx = generate_dummy_appointment() + added_appointment, sig = watcher.add_appointment(appointment) assert added_appointment is True - added_appointment, sig = watcher.add_appointment(create_appointment()) + appointment, dispute_tx = generate_dummy_appointment() + added_appointment = watcher.add_appointment(appointment) assert added_appointment is False @@ -102,7 +117,8 @@ def test_do_subscribe(watcher): zmq_thread.start() try: - block_hash = watcher.block_queue.get(timeout=MAX_APPOINTMENTS) + generate_block() + block_hash = watcher.block_queue.get() assert check_txid_format(block_hash) except Empty: @@ -113,25 +129,28 @@ def test_do_watch(watcher): bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT)) # We will wipe all the previous data and add 5 appointments - watcher.appointments, watcher.locator_uuid_map, txids = create_appointments(APPOINTMENTS) + watcher.appointments, watcher.locator_uuid_map, dispute_txs = create_appointments(APPOINTMENTS) watch_thread = Thread(target=watcher.do_watch) watch_thread.daemon = True watch_thread.start() # Broadcast the first two - for txid in txids[:2]: - bitcoin_cli.sendrawtransaction(txid) + for dispute_tx in dispute_txs[:2]: + r = bitcoin_cli.sendrawtransaction(dispute_tx) # After leaving some time for the block to be mined and processed, the number of appointments should have reduced # by two - sleep(TIME_BETWEEN_BLOCKS*2) + for _ in range(START_TIME_OFFSET + END_TIME_OFFSET): + generate_block() + assert len(watcher.appointments) == APPOINTMENTS - 2 # The rest of appointments will timeout after the end (2) + EXPIRY_DELTA # Wait for an additional block to be safe - sleep((EXPIRY_DELTA + 2 + 1) * TIME_BETWEEN_BLOCKS) + for _ in range(EXPIRY_DELTA + START_TIME_OFFSET + END_TIME_OFFSET): + generate_block() assert len(watcher.appointments) == 0 assert watcher.asleep is True