orderbook: use string values to generate checksum

bfxapi/models/order_book.py

@@ -3,6 +3,7 @@ Module used to describe all of the different data types
 """
 
 import zlib
+import json
 
 class OrderBook:
     """
@@ -31,26 +32,37 @@ class OrderBook:
         """
         return self.asks
 
-    def update_from_snapshot(self, data):
+    def update_from_snapshot(self, data, orig_raw_msg):
         """
         Update the orderbook with a raw orderbook snapshot
         """
-        for order in data:
+        # we need to keep the original string values that are sent to use
-            if len(order) == 4:
+        # this avoids any problems with floats
-                if order[3] < 0:
+        orig_raw = json.loads(orig_raw_msg, parse_float=str, parse_int=str)[1]
+        zip_data = []
+        # zip both the float values and string values together
+        for index, order in enumerate(data):
+            zip_data += [(order, orig_raw[index])]
+        ## build our bids and asks
+        for order in zip_data:
+            if len(order[0]) == 4:
+                if order[0][3] < 0:
                     self.bids += [order]
                 else:
                     self.asks += [order]
             else:
-                if order[2] < 0:
+                if order[0][2] < 0:
                     self.asks += [order]
                 else:
                     self.bids += [order]
 
-    def update_with(self, order):
+    def update_with(self, order, orig_raw_msg):
         """
         Update the orderbook with a single update
         """
+        # keep orginal string vlues to avoid checksum float errors
+        orig_raw = json.loads(orig_raw_msg, parse_float=str, parse_int=str)[1]
+        zip_order = (order, orig_raw)
         if len(order) == 4:
             amount = order[3]
             count = order[2]
@@ -63,12 +75,12 @@ class OrderBook:
 
         # if first item in ordebook
         if len(side) == 0:
-            side += [order]
+            side += [zip_order]
             return
 
-        # match price level
+        # match price level but use the float parsed object
         for index, s_order in enumerate(side):
-            s_price = s_order[0]
+            s_price = s_order[0][0]
             if s_price == price:
                 if count == 0:
                     del side[index]
@@ -81,8 +93,8 @@ class OrderBook:
             return
 
         # add to book and sort lowest to highest
-        side += [order]
+        side += [zip_order]
-        side.sort(key=lambda x: x[0], reverse=not amount < 0)
+        side.sort(key=lambda x: x[0][0], reverse=not amount < 0)
         return
 
     def checksum(self):
@@ -93,17 +105,19 @@ class OrderBook:
         # take set of top 25 bids/asks
         for index in range(0, 25):
             if index < len(self.bids):
-                bid = self.bids[index]
+                # use the string parsed array
+                bid = self.bids[index][1]
                 price = bid[0]
                 amount = bid[3] if len(bid) == 4 else bid[2]
-                data += [str(price)]
+                data += [price]
-                data += [str(amount)]
+                data += [amount]
             if index < len(self.asks):
-                ask = self.asks[index]
+                # use the string parsed array
+                ask = self.asks[index][1]
                 price = ask[0]
                 amount = ask[3] if len(ask) == 4 else ask[2]
-                data += [str(price)]
+                data += [price]
-                data += [str(amount)]
+                data += [amount]
         checksum_str = ':'.join(data)
         # calculate checksum and force signed integer
         checksum = zlib.crc32(checksum_str.encode('utf8')) & 0xffffffff

bfxapi/websockets/BfxWebsocket.py

@@ -98,13 +98,17 @@ class BfxWebsocket(GenericWebsocket):
     }
 
     def __init__(self, API_KEY=None, API_SECRET=None, host='wss://api.bitfinex.com/ws/2',
-                 manageOrderBooks=False, dead_man_switch=False, logLevel='INFO', *args, **kwargs):
+                 manageOrderBooks=False, dead_man_switch=False, logLevel='INFO', parse_float=float,
+                 *args, **kwargs):
         self.API_KEY = API_KEY
         self.API_SECRET = API_SECRET
         self.manageOrderBooks = manageOrderBooks
         self.dead_man_switch = dead_man_switch
         self.pendingOrders = {}
         self.orderBooks = {}
+        # How should we store float values? could also be bfxapi.Decimal
+        # which is slower but has higher precision.
+        self.parse_float = parse_float
 
         super(BfxWebsocket, self).__init__(
             host, logLevel=logLevel, *args, **kwargs)
@@ -149,7 +153,7 @@ class BfxWebsocket(GenericWebsocket):
             self.logger.warn(
                 "Unknown websocket event: '{}' {}".format(eType, msg))
 
-    async def _ws_data_handler(self, data):
+    async def _ws_data_handler(self, data, raw_message_str):
         dataEvent = data[1]
         chan_id = data[0]
 
@@ -161,7 +165,7 @@ class BfxWebsocket(GenericWebsocket):
             if subscription.channel_name == 'candles':
                 await self._candle_handler(data)
             if subscription.channel_name == 'book':
-                await self._order_book_handler(data)
+                await self._order_book_handler(data, raw_message_str)
             if subscription.channel_name == 'trades':
                 await self._trade_handler(data)
         else:
@@ -320,7 +324,7 @@ class BfxWebsocket(GenericWebsocket):
                 data[1], subscription.symbol, subscription.timeframe)
             self._emit('new_candle', candle)
 
-    async def _order_book_handler(self, data):
+    async def _order_book_handler(self, data, orig_raw_message):
         obInfo = data[1]
         chan_id = data[0]
         subscription = self.subscriptionManager.get(data[0])
@@ -345,24 +349,24 @@ class BfxWebsocket(GenericWebsocket):
         isSnapshot = type(obInfo[0]) is list
         if isSnapshot:
             self.orderBooks[symbol] = OrderBook()
-            self.orderBooks[symbol].update_from_snapshot(obInfo)
+            self.orderBooks[symbol].update_from_snapshot(obInfo, orig_raw_message)
             self._emit('order_book_snapshot', {
                        'symbol': symbol, 'data': obInfo})
         else:
-            self.orderBooks[symbol].update_with(obInfo)
+            self.orderBooks[symbol].update_with(obInfo, orig_raw_message)
             self._emit('order_book_update', {'symbol': symbol, 'data': obInfo})
 
     async def on_message(self, message):
         self.logger.debug(message)
         # convert float values to decimal
-        msg = json.loads(message, parse_float=Decimal)
+        msg = json.loads(message, parse_float=self.parse_float)
         self._emit('all', msg)
         if type(msg) is dict:
             # System messages are received as json
             await self._ws_system_handler(msg)
         elif type(msg) is list:
             # All data messages are received as a list
-            await self._ws_data_handler(msg)
+            await self._ws_data_handler(msg, message)
         else:
             self.logger.warn('Unknown websocket response: {}'.format(msg))