From 95759793bae60fbee9fbd4342bd6fe9130b4bb08 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 17 Oct 2019 17:17:52 +0100 Subject: [PATCH] Fixes bitcoin_cli bitcoin_cli as a global variable in the main __init__.py was creating issues related to http.client.CannotSendRequest: Request-sent and connection re-usage. Define a new connection per request. --- pisa/__init__.py | 5 -- pisa/block_processor.py | 10 ++-- pisa/carrier.py | 6 +-- pisa/inspector.py | 1 - pisa/tools.py | 16 ++++-- test/simulator/bitcoind_sim.py | 92 ++++++++++++++++++++++++---------- test/unit/conftest.py | 2 +- test/unit/test_api.py | 4 +- test/unit/test_carrier.py | 2 +- test/unit/test_watcher.py | 2 +- 10 files changed, 90 insertions(+), 50 deletions(-) diff --git a/pisa/__init__.py b/pisa/__init__.py index 5d03345..8aa5441 100644 --- a/pisa/__init__.py +++ b/pisa/__init__.py @@ -11,8 +11,3 @@ logging.basicConfig(format='%(message)s', level=logging.INFO, handlers=[ logging.FileHandler(conf.SERVER_LOG_FILE), logging.StreamHandler() ]) - -# Create RPC connection with bitcoind -# TODO: Check if a long lived connection like this may create problems (timeouts) -bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, - conf.BTC_RPC_PORT)) diff --git a/pisa/block_processor.py b/pisa/block_processor.py index a5f30e3..83b0441 100644 --- a/pisa/block_processor.py +++ b/pisa/block_processor.py @@ -1,8 +1,8 @@ import binascii from hashlib import sha256 -from pisa import bitcoin_cli from pisa.logger import Logger +from pisa.tools import bitcoin_cli from pisa.utils.auth_proxy import JSONRPCException logger = Logger("BlockProcessor") @@ -13,7 +13,7 @@ class BlockProcessor: def get_block(block_hash): try: - block = bitcoin_cli.getblock(block_hash) + block = bitcoin_cli().getblock(block_hash) except JSONRPCException as e: block = None @@ -25,7 +25,7 @@ class BlockProcessor: def get_best_block_hash(): try: - block_hash = bitcoin_cli.getbestblockhash() + block_hash = bitcoin_cli().getbestblockhash() except JSONRPCException as e: block_hash = None @@ -37,7 +37,7 @@ class BlockProcessor: def get_block_count(): try: - block_count = bitcoin_cli.getblockcount() + block_count = bitcoin_cli().getblockcount() except JSONRPCException as e: block_count = None @@ -73,7 +73,7 @@ class BlockProcessor: try: # ToDo: #20-test-tx-decrypting-edge-cases justice_rawtx = appointments[uuid].encrypted_blob.decrypt(dispute_txid) - justice_txid = bitcoin_cli.decoderawtransaction(justice_rawtx).get('txid') + justice_txid = bitcoin_cli().decoderawtransaction(justice_rawtx).get('txid') logger.info("Match found for locator.", locator=locator, uuid=uuid, justice_txid=justice_txid) except JSONRPCException as e: diff --git a/pisa/carrier.py b/pisa/carrier.py index d4cfbed..130cfca 100644 --- a/pisa/carrier.py +++ b/pisa/carrier.py @@ -1,6 +1,6 @@ from pisa.rpc_errors import * -from pisa import bitcoin_cli from pisa.logger import Logger +from pisa.tools import bitcoin_cli from pisa.utils.auth_proxy import JSONRPCException from pisa.errors import UNKNOWN_JSON_RPC_EXCEPTION @@ -20,7 +20,7 @@ class Carrier: def send_transaction(self, rawtx, txid): try: logger.info("Pushing transaction to the network", txid=txid, rawtx=rawtx) - bitcoin_cli.sendrawtransaction(rawtx) + bitcoin_cli().sendrawtransaction(rawtx) receipt = Receipt(delivered=True) @@ -70,7 +70,7 @@ class Carrier: @staticmethod def get_transaction(txid): try: - tx_info = bitcoin_cli.getrawtransaction(txid, 1) + tx_info = bitcoin_cli().getrawtransaction(txid, 1) except JSONRPCException as e: tx_info = None diff --git a/pisa/inspector.py b/pisa/inspector.py index 53055b8..1ae4547 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -2,7 +2,6 @@ import re from pisa import errors import pisa.conf as conf -from pisa import bitcoin_cli from pisa.logger import Logger from pisa.appointment import Appointment from pisa.block_processor import BlockProcessor diff --git a/pisa/tools.py b/pisa/tools.py index aeaa310..adba5e5 100644 --- a/pisa/tools.py +++ b/pisa/tools.py @@ -1,10 +1,15 @@ import re from http.client import HTTPException -from pisa import bitcoin_cli +import pisa.conf as conf from pisa.logger import Logger -from pisa.utils.auth_proxy import JSONRPCException from pisa.rpc_errors import RPC_INVALID_ADDRESS_OR_KEY +from pisa.utils.auth_proxy import AuthServiceProxy, JSONRPCException + + +def bitcoin_cli(): + return AuthServiceProxy("http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, + conf.BTC_RPC_PORT)) # TODO: currently only used in the Responder; might move there or in the BlockProcessor @@ -13,7 +18,7 @@ def check_tx_in_chain(tx_id, logger=Logger(), tx_label='Transaction'): confirmations = 0 try: - tx_info = bitcoin_cli.getrawtransaction(tx_id, 1) + tx_info = bitcoin_cli().getrawtransaction(tx_id, 1) if tx_info.get("confirmations"): confirmations = int(tx_info.get("confirmations")) @@ -38,7 +43,7 @@ def can_connect_to_bitcoind(): can_connect = True try: - bitcoin_cli.help() + bitcoin_cli().help() except (ConnectionRefusedError, JSONRPCException, HTTPException): can_connect = False @@ -50,7 +55,7 @@ def in_correct_network(network): testnet3_genesis_block_hash = "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943" correct_network = False - genesis_block_hash = bitcoin_cli.getblockhash(0) + genesis_block_hash = bitcoin_cli().getblockhash(0) if network == 'mainnet' and genesis_block_hash == mainnet_genesis_block_hash: correct_network = True @@ -65,3 +70,4 @@ def in_correct_network(network): def check_txid_format(txid): # TODO: #12-check-txid-regexp return isinstance(txid, str) and re.search(r'^[0-9A-Fa-f]{64}$', txid) is not None + diff --git a/test/simulator/bitcoind_sim.py b/test/simulator/bitcoind_sim.py index b481b43..3098717 100644 --- a/test/simulator/bitcoind_sim.py +++ b/test/simulator/bitcoind_sim.py @@ -1,21 +1,24 @@ -from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT -from flask import Flask, request, Response, abort -from test.simulator.zmq_publisher import ZMQPublisher -from threading import Thread -from pisa.rpc_errors import * -from pisa.tools import check_txid_format -import logging -import binascii -import json +import re import os import time +import json +import logging +import binascii +from threading import Thread +from flask import Flask, request, Response, abort +from pisa.rpc_errors import * +from test2.simulator.utils import sha256d +from pisa.tools import check_txid_format +from test2.simulator.transaction import TX +from test2.simulator.zmq_publisher import ZMQPublisher +from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT app = Flask(__name__) HOST = 'localhost' PORT = '18443' -TIME_BETWEEN_BLOCKS = 10 +TIME_BETWEEN_BLOCKS = 5 mempool = [] mined_transactions = {} @@ -67,11 +70,11 @@ def process_request(): no_param_err = {"code": RPC_MISC_ERROR, "message": "JSON value is not a {} as expected"} if method == "decoderawtransaction": - txid = get_param(request_data) + rawtx = get_param(request_data) - if isinstance(txid, str): - if check_txid_format(txid): - response["result"] = {"txid": txid} + if isinstance(rawtx, str): + if TX.deserialize(rawtx) is not None: + response["result"] = {"txid": rawtx} else: response["error"] = {"code": RPC_DESERIALIZATION_ERROR, "message": "TX decode failed"} @@ -82,12 +85,12 @@ def process_request(): elif method == "sendrawtransaction": # TODO: A way of rejecting transactions should be added to test edge cases. - txid = get_param(request_data) + rawtx = get_param(request_data) - if isinstance(txid, str): - if check_txid_format(txid): - if txid not in list(mined_transactions.keys()): - mempool.append(txid) + if isinstance(rawtx, str): + if TX.deserialize(rawtx) is not None: + if rawtx not in list(mined_transactions.keys()): + mempool.append(rawtx) else: response["error"] = {"code": RPC_VERIFY_ALREADY_IN_CHAIN, @@ -120,6 +123,8 @@ def process_request(): response["error"] = no_param_err response["error"]["message"] = response["error"]["message"].format("string") + print(response) + elif method == "getblockcount": response["result"] = len(blockchain) @@ -169,6 +174,7 @@ def get_param(request_data): param = None params = request_data.get("params") + if isinstance(params, list) and len(params) > 0: param = params[0] @@ -179,6 +185,33 @@ def load_data(): pass +def create_dummy_transaction(prev_tx_id=None, prev_out_index=None): + tx = TX() + + if prev_tx_id is None: + prev_tx_id = os.urandom(32).hex() + + if prev_out_index is None: + prev_out_index = 0 + + tx.version = 1 + tx.inputs = 1 + tx.outputs = 1 + tx.prev_tx_id = [prev_tx_id] + tx.prev_out_index = [prev_out_index] + tx.nLockTime = 0 + tx.scriptSig = ['47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860' + 'a4acdd12909d831cc56cbbac4622082221a8768d1d0901'] + tx.scriptSig_len = [77] + tx.nSequence = [4294967295] + tx.value = [5000000000] + tx.scriptPubKey = ['4104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c' + '1b7303b8a0626f1baded5c72a704f7e6cd84cac'] + tx.scriptPubKey_len = [67] + + return tx.serialize() + + def simulate_mining(): global mempool, mined_transactions, blocks, blockchain prev_block_hash = None @@ -188,25 +221,32 @@ def simulate_mining(): while True: block_hash = os.urandom(32).hex() - coinbase_tx_hash = os.urandom(32).hex() - txs_to_mine = [coinbase_tx_hash] + coinbase_tx = create_dummy_transaction() + coinbase_tx_hash = sha256d(coinbase_tx) + + txs_to_mine = dict({coinbase_tx_hash: coinbase_tx}) if len(mempool) != 0: # We'll mine up to 100 txs per block - txs_to_mine += mempool[:99] + for rawtx in mempool[:99]: + txid = sha256d(rawtx) + txs_to_mine[txid] = rawtx + mempool = mempool[99:] # Keep track of the mined transaction (to respond to getrawtransaction) - for tx in txs_to_mine: - mined_transactions[tx] = block_hash + for txid, tx in txs_to_mine.items(): + mined_transactions[txid] = {"tx": tx, "block": block_hash} + + blocks[block_hash] = {"tx": list(txs_to_mine.keys()), "height": len(blockchain), + "previousblockhash": prev_block_hash} - blocks[block_hash] = {"tx": txs_to_mine, "height": len(blockchain), "previousblockhash": prev_block_hash} mining_simulator.publish_data(binascii.unhexlify(block_hash)) blockchain.append(block_hash) prev_block_hash = block_hash print("New block mined: {}".format(block_hash)) - print("\tTransactions: {}".format(txs_to_mine)) + print("\tTransactions: {}".format(list(txs_to_mine.keys()))) time.sleep(TIME_BETWEEN_BLOCKS) diff --git a/test/unit/conftest.py b/test/unit/conftest.py index ab6af2b..a8a764c 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -3,7 +3,7 @@ from time import sleep from threading import Thread from pisa.api import start_api -from test.simulator.bitcoind_sim import run_simulator +from test2.simulator.bitcoind_sim import run_simulator @pytest.fixture(scope='session') diff --git a/test/unit/test_api.py b/test/unit/test_api.py index 8a5661d..fd293de 100644 --- a/test/unit/test_api.py +++ b/test/unit/test_api.py @@ -9,7 +9,7 @@ from binascii import unhexlify from apps.cli.blob import Blob from pisa import HOST, PORT, logging from pisa.utils.auth_proxy import AuthServiceProxy -from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS +from test2.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS, create_dummy_transaction from pisa.conf import BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT, MAX_APPOINTMENTS logging.getLogger().disabled = True @@ -25,7 +25,7 @@ def generate_dummy_appointment(dispute_txid): current_height = r.json().get("block_count") - dummy_appointment_data = {"tx": os.urandom(32).hex(), "tx_id": dispute_txid, "start_time": current_height + 5, + dummy_appointment_data = {"tx": create_dummy_transaction(), "tx_id": dispute_txid, "start_time": current_height + 5, "end_time": current_height + 30, "dispute_delta": 20} cipher = "AES-GCM-128" diff --git a/test/unit/test_carrier.py b/test/unit/test_carrier.py index 374dd24..92db193 100644 --- a/test/unit/test_carrier.py +++ b/test/unit/test_carrier.py @@ -5,7 +5,7 @@ from time import sleep from pisa.carrier import Carrier from pisa.rpc_errors import RPC_VERIFY_ALREADY_IN_CHAIN, RPC_DESERIALIZATION_ERROR -from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS +from test2.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS logging.getLogger().disabled = True diff --git a/test/unit/test_watcher.py b/test/unit/test_watcher.py index 9ddcbb5..bc0f5e7 100644 --- a/test/unit/test_watcher.py +++ b/test/unit/test_watcher.py @@ -13,7 +13,7 @@ from pisa.conf import MAX_APPOINTMENTS from pisa.appointment import Appointment from pisa.tools import check_txid_format from pisa.utils.auth_proxy import AuthServiceProxy -from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS +from test2.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS from pisa.conf import EXPIRY_DELTA, BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT logging.getLogger().disabled = True