Fixes bitcoin_cli

bitcoin_cli as a global variable in the main __init__.py was creating issues related to http.client.CannotSendRequest: Request-sent and connection re-usage. Define a new connection per request.

pisa/__init__.py

@@ -11,8 +11,3 @@ logging.basicConfig(format='%(message)s', level=logging.INFO, handlers=[
     logging.FileHandler(conf.SERVER_LOG_FILE),
     logging.StreamHandler()
 ])
-
-# Create RPC connection with bitcoind
-# TODO: Check if a long lived connection like this may create problems (timeouts)
-bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST,
-                                                       conf.BTC_RPC_PORT))

pisa/block_processor.py

@@ -1,8 +1,8 @@
 import binascii
 from hashlib import sha256
 
-from pisa import bitcoin_cli
 from pisa.logger import Logger
+from pisa.tools import bitcoin_cli
 from pisa.utils.auth_proxy import JSONRPCException
 
 logger = Logger("BlockProcessor")
@@ -13,7 +13,7 @@ class BlockProcessor:
     def get_block(block_hash):
 
         try:
-            block = bitcoin_cli.getblock(block_hash)
+            block = bitcoin_cli().getblock(block_hash)
 
         except JSONRPCException as e:
             block = None
@@ -25,7 +25,7 @@ class BlockProcessor:
     def get_best_block_hash():
 
         try:
-            block_hash = bitcoin_cli.getbestblockhash()
+            block_hash = bitcoin_cli().getbestblockhash()
 
         except JSONRPCException as e:
             block_hash = None
@@ -37,7 +37,7 @@ class BlockProcessor:
     def get_block_count():
 
         try:
-            block_count = bitcoin_cli.getblockcount()
+            block_count = bitcoin_cli().getblockcount()
 
         except JSONRPCException as e:
             block_count = None
@@ -73,7 +73,7 @@ class BlockProcessor:
                 try:
                     # ToDo: #20-test-tx-decrypting-edge-cases
                     justice_rawtx = appointments[uuid].encrypted_blob.decrypt(dispute_txid)
-                    justice_txid = bitcoin_cli.decoderawtransaction(justice_rawtx).get('txid')
+                    justice_txid = bitcoin_cli().decoderawtransaction(justice_rawtx).get('txid')
                     logger.info("Match found for locator.", locator=locator, uuid=uuid, justice_txid=justice_txid)
 
                 except JSONRPCException as e:

pisa/carrier.py

@@ -1,6 +1,6 @@
 from pisa.rpc_errors import *
-from pisa import bitcoin_cli
 from pisa.logger import Logger
+from pisa.tools import bitcoin_cli
 from pisa.utils.auth_proxy import JSONRPCException
 from pisa.errors import UNKNOWN_JSON_RPC_EXCEPTION
 
@@ -20,7 +20,7 @@ class Carrier:
     def send_transaction(self, rawtx, txid):
         try:
             logger.info("Pushing transaction to the network", txid=txid, rawtx=rawtx)
-            bitcoin_cli.sendrawtransaction(rawtx)
+            bitcoin_cli().sendrawtransaction(rawtx)
 
             receipt = Receipt(delivered=True)
 
@@ -70,7 +70,7 @@ class Carrier:
     @staticmethod
     def get_transaction(txid):
         try:
-            tx_info = bitcoin_cli.getrawtransaction(txid, 1)
+            tx_info = bitcoin_cli().getrawtransaction(txid, 1)
 
         except JSONRPCException as e:
             tx_info = None

pisa/inspector.py

@@ -2,7 +2,6 @@ import re
 
 from pisa import errors
 import pisa.conf as conf
-from pisa import bitcoin_cli
 from pisa.logger import Logger
 from pisa.appointment import Appointment
 from pisa.block_processor import BlockProcessor

pisa/tools.py

@@ -1,10 +1,15 @@
 import re
 from http.client import HTTPException
 
-from pisa import bitcoin_cli
+import pisa.conf as conf
 from pisa.logger import Logger
-from pisa.utils.auth_proxy import JSONRPCException
 from pisa.rpc_errors import RPC_INVALID_ADDRESS_OR_KEY
+from pisa.utils.auth_proxy import AuthServiceProxy, JSONRPCException
+
+
+def bitcoin_cli():
+    return AuthServiceProxy("http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST,
+                                                    conf.BTC_RPC_PORT))
 
 
 # TODO: currently only used in the Responder; might move there or in the BlockProcessor
@@ -13,7 +18,7 @@ def check_tx_in_chain(tx_id, logger=Logger(), tx_label='Transaction'):
     confirmations = 0
 
     try:
-        tx_info = bitcoin_cli.getrawtransaction(tx_id, 1)
+        tx_info = bitcoin_cli().getrawtransaction(tx_id, 1)
 
         if tx_info.get("confirmations"):
             confirmations = int(tx_info.get("confirmations"))
@@ -38,7 +43,7 @@ def can_connect_to_bitcoind():
     can_connect = True
 
     try:
-        bitcoin_cli.help()
+        bitcoin_cli().help()
     except (ConnectionRefusedError, JSONRPCException, HTTPException):
         can_connect = False
 
@@ -50,7 +55,7 @@ def in_correct_network(network):
     testnet3_genesis_block_hash = "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"
     correct_network = False
 
-    genesis_block_hash = bitcoin_cli.getblockhash(0)
+    genesis_block_hash = bitcoin_cli().getblockhash(0)
 
     if network == 'mainnet' and genesis_block_hash == mainnet_genesis_block_hash:
         correct_network = True
@@ -65,3 +70,4 @@ def in_correct_network(network):
 def check_txid_format(txid):
     # TODO: #12-check-txid-regexp
     return isinstance(txid, str) and re.search(r'^[0-9A-Fa-f]{64}$', txid) is not None
+

test/simulator/bitcoind_sim.py

@@ -1,21 +1,24 @@
-from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT
+import re
-from flask import Flask, request, Response, abort
-from test.simulator.zmq_publisher import ZMQPublisher
-from threading import Thread
-from pisa.rpc_errors import *
-from pisa.tools import check_txid_format
-import logging
-import binascii
-import json
 import os
 import time
+import json
+import logging
+import binascii
+from threading import Thread
+from flask import Flask, request, Response, abort
 
+from pisa.rpc_errors import *
+from test2.simulator.utils import sha256d
+from pisa.tools import check_txid_format
+from test2.simulator.transaction import TX
+from test2.simulator.zmq_publisher import ZMQPublisher
+from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT
 
 app = Flask(__name__)
 HOST = 'localhost'
 PORT = '18443'
 
-TIME_BETWEEN_BLOCKS = 10
+TIME_BETWEEN_BLOCKS = 5
 
 mempool = []
 mined_transactions = {}
@@ -67,11 +70,11 @@ def process_request():
     no_param_err = {"code": RPC_MISC_ERROR, "message": "JSON value is not a {} as expected"}
 
     if method == "decoderawtransaction":
-        txid = get_param(request_data)
+        rawtx = get_param(request_data)
 
-        if isinstance(txid, str):
+        if isinstance(rawtx, str):
-            if check_txid_format(txid):
+            if TX.deserialize(rawtx) is not None:
-                response["result"] = {"txid": txid}
+                response["result"] = {"txid": rawtx}
 
             else:
                 response["error"] = {"code": RPC_DESERIALIZATION_ERROR, "message": "TX decode failed"}
@@ -82,12 +85,12 @@ def process_request():
 
     elif method == "sendrawtransaction":
         # TODO: A way of rejecting transactions should be added to test edge cases.
-        txid = get_param(request_data)
+        rawtx = get_param(request_data)
 
-        if isinstance(txid, str):
+        if isinstance(rawtx, str):
-            if check_txid_format(txid):
+            if TX.deserialize(rawtx) is not None:
-                if txid not in list(mined_transactions.keys()):
+                if rawtx not in list(mined_transactions.keys()):
-                    mempool.append(txid)
+                    mempool.append(rawtx)
 
                 else:
                     response["error"] = {"code": RPC_VERIFY_ALREADY_IN_CHAIN,
@@ -120,6 +123,8 @@ def process_request():
             response["error"] = no_param_err
             response["error"]["message"] = response["error"]["message"].format("string")
 
+        print(response)
+
     elif method == "getblockcount":
         response["result"] = len(blockchain)
 
@@ -169,6 +174,7 @@ def get_param(request_data):
     param = None
 
     params = request_data.get("params")
+
     if isinstance(params, list) and len(params) > 0:
         param = params[0]
 
@@ -179,6 +185,33 @@ def load_data():
     pass
 
 
+def create_dummy_transaction(prev_tx_id=None, prev_out_index=None):
+    tx = TX()
+
+    if prev_tx_id is None:
+        prev_tx_id = os.urandom(32).hex()
+
+    if prev_out_index is None:
+        prev_out_index = 0
+
+    tx.version = 1
+    tx.inputs = 1
+    tx.outputs = 1
+    tx.prev_tx_id = [prev_tx_id]
+    tx.prev_out_index = [prev_out_index]
+    tx.nLockTime = 0
+    tx.scriptSig = ['47304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860'
+                    'a4acdd12909d831cc56cbbac4622082221a8768d1d0901']
+    tx.scriptSig_len = [77]
+    tx.nSequence = [4294967295]
+    tx.value = [5000000000]
+    tx.scriptPubKey = ['4104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c'
+                       '1b7303b8a0626f1baded5c72a704f7e6cd84cac']
+    tx.scriptPubKey_len = [67]
+
+    return tx.serialize()
+
+
 def simulate_mining():
     global mempool, mined_transactions, blocks, blockchain
     prev_block_hash = None
@@ -188,25 +221,32 @@ def simulate_mining():
 
     while True:
         block_hash = os.urandom(32).hex()
-        coinbase_tx_hash = os.urandom(32).hex()
+        coinbase_tx = create_dummy_transaction()
-        txs_to_mine = [coinbase_tx_hash]
+        coinbase_tx_hash = sha256d(coinbase_tx)
+
+        txs_to_mine = dict({coinbase_tx_hash: coinbase_tx})
 
         if len(mempool) != 0:
             # We'll mine up to 100 txs per block
-            txs_to_mine += mempool[:99]
+            for rawtx in mempool[:99]:
+                txid = sha256d(rawtx)
+                txs_to_mine[txid] = rawtx
+
             mempool = mempool[99:]
 
         # Keep track of the mined transaction (to respond to getrawtransaction)
-        for tx in txs_to_mine:
+        for txid, tx in txs_to_mine.items():
-            mined_transactions[tx] = block_hash
+            mined_transactions[txid] = {"tx": tx, "block": block_hash}
+
+        blocks[block_hash] = {"tx": list(txs_to_mine.keys()), "height": len(blockchain),
+                              "previousblockhash": prev_block_hash}
 
-        blocks[block_hash] = {"tx": txs_to_mine, "height": len(blockchain), "previousblockhash": prev_block_hash}
         mining_simulator.publish_data(binascii.unhexlify(block_hash))
         blockchain.append(block_hash)
         prev_block_hash = block_hash
 
         print("New block mined: {}".format(block_hash))
-        print("\tTransactions: {}".format(txs_to_mine))
+        print("\tTransactions: {}".format(list(txs_to_mine.keys())))
 
         time.sleep(TIME_BETWEEN_BLOCKS)
 

test/unit/conftest.py

@@ -3,7 +3,7 @@ from time import sleep
 from threading import Thread
 
 from pisa.api import start_api
-from test.simulator.bitcoind_sim import run_simulator
+from test2.simulator.bitcoind_sim import run_simulator
 
 
 @pytest.fixture(scope='session')

test/unit/test_api.py

@@ -9,7 +9,7 @@ from binascii import unhexlify
 from apps.cli.blob import Blob
 from pisa import HOST, PORT, logging
 from pisa.utils.auth_proxy import AuthServiceProxy
-from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS
+from test2.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS, create_dummy_transaction
 from pisa.conf import BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT, MAX_APPOINTMENTS
 
 logging.getLogger().disabled = True
@@ -25,7 +25,7 @@ def generate_dummy_appointment(dispute_txid):
 
     current_height = r.json().get("block_count")
 
-    dummy_appointment_data = {"tx": os.urandom(32).hex(), "tx_id": dispute_txid, "start_time": current_height + 5,
+    dummy_appointment_data = {"tx": create_dummy_transaction(), "tx_id": dispute_txid, "start_time": current_height + 5,
                               "end_time": current_height + 30, "dispute_delta": 20}
 
     cipher = "AES-GCM-128"

test/unit/test_carrier.py

@@ -5,7 +5,7 @@ from time import sleep
 
 from pisa.carrier import Carrier
 from pisa.rpc_errors import RPC_VERIFY_ALREADY_IN_CHAIN, RPC_DESERIALIZATION_ERROR
-from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS
+from test2.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS
 
 logging.getLogger().disabled = True
 

test/unit/test_watcher.py

@@ -13,7 +13,7 @@ from pisa.conf import MAX_APPOINTMENTS
 from pisa.appointment import Appointment
 from pisa.tools import check_txid_format
 from pisa.utils.auth_proxy import AuthServiceProxy
-from test.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS
+from test2.simulator.bitcoind_sim import TIME_BETWEEN_BLOCKS
 from pisa.conf import EXPIRY_DELTA, BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT
 
 logging.getLogger().disabled = True