Merge pull request #28 from sr-gi/27-zmq-bitcoinrpc-simulators

Created a tinny bitcoind simulator with zmq pub capabilities

pisa/tools.py

@@ -1,3 +1,4 @@
+import re
 from pisa.utils.authproxy import JSONRPCException
 from pisa.rpc_errors import RPC_INVALID_ADDRESS_OR_KEY
 from http.client import HTTPException
@@ -55,3 +56,7 @@ def in_correct_network(bitcoin_cli, network):
 
     return correct_network
 
+
+def check_txid_format(txid):
+    # TODO: #12-check-txid-regexp
+    return isinstance(txid, str) and re.search(r'^[0-9A-Fa-f]{64}$', txid) is not None

tests/simulator/appointment_test.py

@@ -0,0 +1,52 @@
+import os
+import json
+import requests
+import time
+from hashlib import sha256
+from binascii import hexlify, unhexlify
+from apps.cli.blob import Blob
+from pisa import HOST, PORT
+from pisa.utils.authproxy import AuthServiceProxy
+from pisa.conf import BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT
+
+PISA_API = "http://{}:{}".format(HOST, PORT)
+
+
+def generate_dummy_appointment(dispute_txid):
+    r = requests.get(url=PISA_API+'/get_block_count', timeout=5)
+
+    current_height = r.json().get("block_count")
+
+    dummy_appointment_data = {"tx": hexlify(os.urandom(32)).decode('utf-8'),
+                              "tx_id": dispute_txid, "start_time": current_height + 5,
+                              "end_time": current_height + 10, "dispute_delta": 20}
+
+    cipher = "AES-GCM-128"
+    hash_function = "SHA256"
+
+    locator = sha256(unhexlify(dummy_appointment_data.get("tx_id"))).hexdigest()
+    blob = Blob(dummy_appointment_data.get("tx"), cipher, hash_function)
+
+    encrypted_blob = blob.encrypt((dummy_appointment_data.get("tx_id")), debug=False, logging=False)
+
+    appointment = {"locator": locator, "start_time": dummy_appointment_data.get("start_time"),
+                   "end_time": dummy_appointment_data.get("end_time"),
+                   "dispute_delta": dummy_appointment_data.get("dispute_delta"),
+                   "encrypted_blob": encrypted_blob, "cipher": cipher, "hash_function": hash_function}
+
+    return appointment
+
+
+dispute_txid = hexlify(os.urandom(32)).decode('utf-8')
+appointment = generate_dummy_appointment(dispute_txid)
+
+print("Sending appointment (locator: {}) to PISA".format(appointment.get("locator")))
+r = requests.post(url=PISA_API, json=json.dumps(appointment), timeout=5)
+print(r, r.reason)
+
+print("Sleeping 10 sec")
+time.sleep(10)
+bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT))
+
+print("Triggering PISA with dispute tx")
+bitcoin_cli.sendrawtransaction(dispute_txid)

tests/simulator/bitcoin_sim_tests.py

@@ -0,0 +1,109 @@
+import os
+import binascii
+from pisa.utils.authproxy import AuthServiceProxy, JSONRPCException
+from pisa.conf import BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT
+from pisa.tools import check_txid_format
+
+
+bitcoin_cli = AuthServiceProxy("http://%s:%s@%s:%d" % (BTC_RPC_USER, BTC_RPC_PASSWD, BTC_RPC_HOST, BTC_RPC_PORT))
+
+# Help should always return 0
+assert(bitcoin_cli.help() == 0)
+
+# getblockhash should return a blockid (which matches the txid format)
+block_hash = bitcoin_cli.getblockhash(0)
+assert(check_txid_format(block_hash))
+
+# Check that the values are within range and of the proper format (all should fail)
+values = [-1, 500, None, '', '111', [], 1.1]
+print("getblockhash fails ({}):".format(len(values)))
+
+for v in values:
+    try:
+        block_hash = bitcoin_cli.getblockhash(v)
+        assert False
+    except JSONRPCException as e:
+        print('\t{}'.format(e))
+
+# getblock should return a list of transactions and the height
+block = bitcoin_cli.getblock(block_hash)
+assert(isinstance(block.get('tx'), list))
+assert(len(block.get('tx')) != 0)
+assert(isinstance(block.get('height'), int))
+
+# Some fails
+values += ["a"*64, binascii.hexlify(os.urandom(32)).decode()]
+print("\ngetblock fails ({}):".format(len(values)))
+
+for v in values:
+    try:
+        block = bitcoin_cli.getblock(v)
+        assert False
+    except JSONRPCException as e:
+        print('\t{}'.format(e))
+
+# decoderawtransaction should only return if the given transaction matches a txid format
+coinbase_tx = block.get('tx')[0]
+tx = bitcoin_cli.decoderawtransaction(coinbase_tx)
+assert(isinstance(tx, dict))
+assert(isinstance(tx.get('txid'), str))
+assert(check_txid_format(tx.get('txid')))
+
+# Therefore should also work for a random formatted 32-byte hex in our simulation
+random_tx = binascii.hexlify(os.urandom(32)).decode()
+tx = bitcoin_cli.decoderawtransaction(random_tx)
+assert(isinstance(tx, dict))
+assert(isinstance(tx.get('txid'), str))
+assert(check_txid_format(tx.get('txid')))
+
+# But it should fail for not proper formatted one
+values = [1, None, '', "a"*63, "b"*65, [], binascii.hexlify(os.urandom(31)).hex()]
+print("\ndecoderawtransaction fails ({}):".format(len(values)))
+
+for v in values:
+    try:
+        block = bitcoin_cli.decoderawtransaction(v)
+        assert False
+    except JSONRPCException as e:
+        print('\t{}'.format(e))
+
+# sendrawtransaction should only allow txids that the simulator has not mined yet
+bitcoin_cli.sendrawtransaction(binascii.hexlify(os.urandom(32)).decode())
+
+# Any data not matching the txid format or that matches with an already mined transaction should fail
+values += [coinbase_tx]
+
+print("\nsendrawtransaction fails ({}):".format(len(values)))
+
+for v in values:
+    try:
+        block = bitcoin_cli.sendrawtransaction(v)
+        assert False
+    except JSONRPCException as e:
+        print('\t{}'.format(e))
+
+# getrawtransaction should work for existing transactions, and fail for non-existing ones
+tx = bitcoin_cli.getrawtransaction(coinbase_tx)
+
+assert(isinstance(tx, dict))
+assert(isinstance(tx.get('confirmations'), int))
+
+print("\nsendrawtransaction fails ({}):".format(len(values)))
+
+for v in values:
+    try:
+        block = bitcoin_cli.sendrawtransaction(v)
+        assert False
+    except JSONRPCException as e:
+        print('\t{}'.format(e))
+
+# getblockcount should always return a positive integer
+bc = bitcoin_cli.getblockcount()
+assert (isinstance(bc, int))
+assert (bc >= 0)
+
+print("\nAll tests passed!")
+
+
+
+

tests/simulator/bitcoind_sim.py

@@ -0,0 +1,211 @@
+from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT
+from flask import Flask, request, Response, abort
+from tests.simulator.zmq_publisher import ZMQPublisher
+from threading import Thread
+from pisa.rpc_errors import *
+from pisa.tools import check_txid_format
+import logging
+import binascii
+import json
+import os
+import time
+
+
+app = Flask(__name__)
+HOST = 'localhost'
+PORT = '18443'
+
+
+@app.route('/', methods=['POST'])
+def process_request():
+    """
+    process_requests simulates the bitcoin-rpc server run by bitcoind. The available commands are limited to the ones
+    we'll need to use in pisa. The model we will be using is pretty simplified to reduce the complexity of simulating
+    bitcoind:
+
+    Raw transactions:       raw transactions will actually be transaction ids (txids). Pisa will, therefore, receive
+                            encrypted blobs that encrypt ids instead of real transactions.
+
+    decoderawtransaction:   querying for the decoding of a raw transaction will return a dictionary with a single
+                            field: "txid", which will match with the txid provided in the request
+
+    sendrawtransaction:     sending a rawtransaction will notify our mining simulator to include such transaction in a
+                            subsequent block.
+
+    getrawtransaction:      requesting a rawtransaction from a txid will return a dictionary containing a single field:
+                            "confirmations", since rawtransactions are only queried to check whether a transaction has
+                            made it to a block or not.
+
+    getblockcount:          the block count will be get from the mining simulator by querying how many blocks have been
+                            emited so far.
+
+    getblock:               querying for a block will return a dictionary with a three fields: "tx" representing a list
+                            of transactions, "height" representing the block height and "hash" representing the block
+                            hash. Both will be got from the mining simulator.
+
+    getblockhash:           a block hash is only queried by pisad on bootstrapping to check the network bitcoind is
+                            running on.
+
+    help:                   help is only used as a sample command to test if bitcoind is running when bootstrapping
+                            pisad. It will return a 200/OK with no data.
+    """
+
+    global mempool
+    request_data = request.get_json()
+    method = request_data.get('method')
+
+    response = {"id": 0, "result": 0, "error": None}
+    no_param_err = {"code": RPC_MISC_ERROR, "message": "JSON value is not a {} as expected"}
+
+    if method == "decoderawtransaction":
+        txid = get_param(request_data)
+
+        if isinstance(txid, str):
+            if check_txid_format(txid):
+                response["result"] = {"txid": txid}
+
+            else:
+                response["error"] = {"code": RPC_DESERIALIZATION_ERROR, "message": "TX decode failed"}
+
+        else:
+            response["error"] = no_param_err
+            response["error"]["message"] = response["error"]["message"].format("string")
+
+    elif method == "sendrawtransaction":
+        # TODO: A way of rejecting transactions should be added to test edge cases.
+        txid = get_param(request_data)
+
+        if isinstance(txid, str):
+            if check_txid_format(txid):
+                if txid not in mempool and txid not in list(mined_transactions.keys()):
+                    mempool.append(txid)
+
+                else:
+                    response["error"] = {"code": RPC_VERIFY_ALREADY_IN_CHAIN,
+                                         "message": "Transaction already in block chain"}
+
+            else:
+                response["error"] = {"code": RPC_DESERIALIZATION_ERROR, "message": "TX decode failed"}
+
+        else:
+            response["error"] = no_param_err
+            response["error"]["message"] = response["error"]["message"].format("string")
+
+    elif method == "getrawtransaction":
+        txid = get_param(request_data)
+
+        if isinstance(txid, str):
+            block = blocks.get(mined_transactions.get(txid))
+
+            if block:
+                response["result"] = {"confirmations": len(blockchain) - block.get('height')}
+
+            else:
+                response["error"] = {'code': RPC_INVALID_ADDRESS_OR_KEY,
+                                     'message': 'No such mempool or blockchain transaction. Use gettransaction for '
+                                                'wallet transactions.'}
+        else:
+            response["error"] = no_param_err
+            response["error"]["message"] = response["error"]["message"].format("string")
+
+    elif method == "getblockcount":
+        response["result"] = len(blockchain)
+
+    elif method == "getblock":
+        blockid = get_param(request_data)
+
+        if isinstance(blockid, str):
+            block = blocks.get(blockid)
+
+            if block:
+                block["hash"] = blockid
+                response["result"] = block
+
+            else:
+                response["error"] = {"code": RPC_INVALID_ADDRESS_OR_KEY, "message": "Block not found"}
+
+        else:
+            response["error"] = no_param_err
+            response["error"]["message"] = response["error"]["message"].format("string")
+
+    elif method == "getblockhash":
+        height = get_param(request_data)
+
+        if isinstance(height, int):
+            if 0 <= height <= len(blockchain):
+                response["result"] = blockchain[height]
+
+            else:
+                response["error"] = {"code": RPC_INVALID_PARAMETER, "message": "Block height out of range"}
+        else:
+            response["error"] = no_param_err
+            response["error"]["message"] = response["error"]["message"].format("integer")
+
+    elif method == "help":
+        pass
+
+    else:
+        return abort(404, "Method not found")
+
+    return Response(json.dumps(response), status=200, mimetype='application/json')
+
+
+def get_param(request_data):
+    param = None
+
+    params = request_data.get("params")
+    if isinstance(params, list) and len(params) > 0:
+        param = params[0]
+
+    return param
+
+
+def load_data():
+    pass
+
+
+def simulate_mining():
+    global mempool, mined_transactions, blocks, blockchain
+    prev_block_hash = None
+
+    while True:
+        block_hash = binascii.hexlify(os.urandom(32)).decode('utf-8')
+        coinbase_tx_hash = binascii.hexlify(os.urandom(32)).decode('utf-8')
+        txs_to_mine = [coinbase_tx_hash]
+
+        if len(mempool) != 0:
+            # We'll mine up to 100 txs per block
+            txs_to_mine += mempool[:99]
+            mempool = mempool[99:]
+
+        # Keep track of the mined transaction (to respond to getrawtransaction)
+        for tx in txs_to_mine:
+            mined_transactions[tx] = block_hash
+
+        blocks[block_hash] = {"tx": txs_to_mine, "height": len(blockchain), "previousblockhash": prev_block_hash}
+        mining_simulator.publish_data(binascii.unhexlify(block_hash))
+        blockchain.append(block_hash)
+        prev_block_hash = block_hash
+
+        print("New block mined: {}".format(block_hash))
+        print("\tTransactions: {}".format(txs_to_mine))
+
+        time.sleep(10)
+
+
+if __name__ == '__main__':
+    mining_simulator = ZMQPublisher(topic=b'hashblock', feed_protocol=FEED_PROTOCOL, feed_addr=FEED_ADDR,
+                                    feed_port=FEED_PORT)
+
+    mempool = []
+    mined_transactions = {}
+    blocks = {}
+    blockchain = []
+
+    mining_thread = Thread(target=simulate_mining)
+    mining_thread.start()
+
+    # Setting Flask log to ERROR only so it does not mess with out logging
+    logging.getLogger('werkzeug').setLevel(logging.ERROR)
+
+    app.run(host=HOST, port=PORT)

tests/simulator/zmq_publisher.py

@@ -0,0 +1,12 @@
+import zmq
+
+
+class ZMQPublisher:
+    def __init__(self, topic, feed_protocol, feed_addr, feed_port):
+        self.topic = topic
+        self.context = zmq.Context()
+        self.socket = self.context.socket(zmq.PUB)
+        self.socket.bind("%s://%s:%s" % (feed_protocol, feed_addr, feed_port))
+
+    def publish_data(self, data):
+        self.socket.send_multipart([self.topic, data])