From 59a0dca66eed488fd5a586f570c99f0431cfaee1 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 19 May 2023 15:37:03 +0200 Subject: [PATCH 01/65] Improve and rewrite bfxapi.websocket.subscriptions. --- bfxapi/websocket/subscriptions.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bfxapi/websocket/subscriptions.py b/bfxapi/websocket/subscriptions.py index 233becc..72df007 100644 --- a/bfxapi/websocket/subscriptions.py +++ b/bfxapi/websocket/subscriptions.py @@ -1,16 +1,18 @@ from typing import TypedDict, Union, Literal, Optional -__all__ = [ - "Subscription", - - "Ticker", - "Trades", - "Book", - "Candles", - "Status" +_Channel = Literal[ + "ticker", + "trades", + "book", + "candles", + "status" ] -_Header = TypedDict("_Header", { "event": Literal["subscribed"], "channel": str, "chanId": int }) +_Header = TypedDict("_Header", { + "event": Literal["subscribed"], + "channel": _Channel, + "chanId": int +}) Subscription = Union[_Header, "Ticker", "Trades", "Book", "Candles", "Status"] From 57680abd06eeb05cccbf4d8193859dd7e22849eb Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 19 May 2023 15:43:35 +0200 Subject: [PATCH 02/65] Rename bfxapi.websocket.handlers.authenticated_events_handler to auth_events_handler (AuthenticatedEventsHandler -> AuthEventsHandler). --- bfxapi/websocket/client/bfx_websocket_client.py | 8 ++++---- bfxapi/websocket/handlers/__init__.py | 2 +- ...enticated_events_handler.py => auth_events_handler.py} | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) rename bfxapi/websocket/handlers/{authenticated_events_handler.py => auth_events_handler.py} (93%) diff --git a/bfxapi/websocket/client/bfx_websocket_client.py b/bfxapi/websocket/client/bfx_websocket_client.py index e0f90e2..c59f1b1 100644 --- a/bfxapi/websocket/client/bfx_websocket_client.py +++ b/bfxapi/websocket/client/bfx_websocket_client.py @@ -11,7 +11,7 @@ from pyee.asyncio import AsyncIOEventEmitter from .bfx_websocket_bucket import _HEARTBEAT, F, _require_websocket_connection, BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs -from ..handlers import PublicChannelsHandler, AuthenticatedEventsHandler +from ..handlers import PublicChannelsHandler, AuthEventsHandler from ..exceptions import WebSocketAuthenticationRequired, InvalidAuthenticationCredentials, EventNotSupported, \ ZeroConnectionsError, ReconnectionTimeoutError, OutdatedClientVersion @@ -57,14 +57,14 @@ class BfxWebSocketClient: ONCE_EVENTS = [ "open", "authenticated", "disconnection", - *AuthenticatedEventsHandler.ONCE_EVENTS + *AuthEventsHandler.ONCE_EVENTS ] EVENTS = [ "subscribed", "wss-error", *ONCE_EVENTS, *PublicChannelsHandler.EVENTS, - *AuthenticatedEventsHandler.ON_EVENTS + *AuthEventsHandler.ON_EVENTS ] def __init__(self, host, credentials, *, wss_timeout = 60 * 15, log_filename = None, log_level = "INFO"): @@ -76,7 +76,7 @@ class BfxWebSocketClient: self.event_emitter = AsyncIOEventEmitter() - self.handler = AuthenticatedEventsHandler(event_emitter=self.event_emitter) + self.handler = AuthEventsHandler(event_emitter=self.event_emitter) self.inputs = BfxWebSocketInputs(handle_websocket_input=self.__handle_websocket_input) diff --git a/bfxapi/websocket/handlers/__init__.py b/bfxapi/websocket/handlers/__init__.py index 98dadbb..23f5aad 100644 --- a/bfxapi/websocket/handlers/__init__.py +++ b/bfxapi/websocket/handlers/__init__.py @@ -1,2 +1,2 @@ from .public_channels_handler import PublicChannelsHandler -from .authenticated_events_handler import AuthenticatedEventsHandler +from .auth_events_handler import AuthEventsHandler diff --git a/bfxapi/websocket/handlers/authenticated_events_handler.py b/bfxapi/websocket/handlers/auth_events_handler.py similarity index 93% rename from bfxapi/websocket/handlers/authenticated_events_handler.py rename to bfxapi/websocket/handlers/auth_events_handler.py index b3e1e12..701e395 100644 --- a/bfxapi/websocket/handlers/authenticated_events_handler.py +++ b/bfxapi/websocket/handlers/auth_events_handler.py @@ -2,7 +2,7 @@ from ...types import serializers from ...types.serializers import _Notification -class AuthenticatedEventsHandler: +class AuthEventsHandler: __once_abbreviations = { "os": "order_snapshot", "ps": "position_snapshot", "fos": "funding_offer_snapshot", "fcs": "funding_credit_snapshot", "fls": "funding_loan_snapshot", "ws": "wallet_snapshot" @@ -49,9 +49,9 @@ class AuthenticatedEventsHandler: if abbrevation == "n": return self.__notification(stream) - for abbrevations, serializer in AuthenticatedEventsHandler.__serializers.items(): + for abbrevations, serializer in AuthEventsHandler.__serializers.items(): if abbrevation in abbrevations: - event = AuthenticatedEventsHandler.__abbreviations[abbrevation] + event = AuthEventsHandler.__abbreviations[abbrevation] if all(isinstance(substream, list) for substream in stream): return self.event_emitter.emit(event, [ serializer.parse(*substream) for substream in stream ]) From c8290f144bf5f40203a0ce834a90340dcbc23689 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 19 May 2023 22:13:15 +0200 Subject: [PATCH 03/65] Upgrade to Mypy 1.3.0 (old: 0.991). Fix compatibility problems with Mypy. Add type hints to bfxapi.websocket.handlers. --- .../rest/endpoints/rest_public_endpoints.py | 6 +- bfxapi/types/serializers.py | 3 +- .../websocket/client/bfx_websocket_bucket.py | 2 +- .../websocket/handlers/auth_events_handler.py | 62 ++++--- .../handlers/public_channels_handler.py | 172 +++++++++--------- bfxapi/websocket/subscriptions.py | 49 +++-- dev-requirements.txt | Bin 600 -> 600 bytes 7 files changed, 161 insertions(+), 133 deletions(-) diff --git a/bfxapi/rest/endpoints/rest_public_endpoints.py b/bfxapi/rest/endpoints/rest_public_endpoints.py index 99cb725..e1c20ff 100644 --- a/bfxapi/rest/endpoints/rest_public_endpoints.py +++ b/bfxapi/rest/endpoints/rest_public_endpoints.py @@ -262,9 +262,9 @@ class RestPublicEndpoints(Middleware): limit: Optional[int] = None) -> List[PulseMessage]: messages = [] - for subdata in self._get("pulse/hist", params={ "end": end, "limit": limit }): - subdata[18] = subdata[18][0] - message = serializers.PulseMessage.parse(*subdata) + for sub_data in self._get("pulse/hist", params={ "end": end, "limit": limit }): + sub_data[18] = sub_data[18][0] + message = serializers.PulseMessage.parse(*sub_data) messages.append(message) return messages diff --git a/bfxapi/types/serializers.py b/bfxapi/types/serializers.py index f853ce4..f7838bf 100644 --- a/bfxapi/types/serializers.py +++ b/bfxapi/types/serializers.py @@ -1,6 +1,7 @@ from .import dataclasses -from .labeler import \ +#pylint: disable-next=unused-import +from .labeler import _Serializer, \ generate_labeler_serializer, generate_recursive_serializer #pylint: disable-next=unused-import diff --git a/bfxapi/websocket/client/bfx_websocket_bucket.py b/bfxapi/websocket/client/bfx_websocket_bucket.py index 500e9db..927a3e4 100644 --- a/bfxapi/websocket/client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/client/bfx_websocket_bucket.py @@ -62,7 +62,7 @@ class BfxWebSocketBucket: if isinstance(message, list): if (chan_id := message[0]) and message[1] != _HEARTBEAT: - self.handler.handle(self.subscriptions[chan_id], *message[1:]) + self.handler.handle(self.subscriptions[chan_id], message[1:]) try: await _connection() diff --git a/bfxapi/websocket/handlers/auth_events_handler.py b/bfxapi/websocket/handlers/auth_events_handler.py index 701e395..8c5cab3 100644 --- a/bfxapi/websocket/handlers/auth_events_handler.py +++ b/bfxapi/websocket/handlers/auth_events_handler.py @@ -1,6 +1,15 @@ -from ...types import serializers +from typing import TYPE_CHECKING, \ + Union, Dict, Tuple, Any -from ...types.serializers import _Notification +from bfxapi.types import serializers + +from bfxapi.types.serializers import _Notification + +if TYPE_CHECKING: + from bfxapi.types.dataclasses import \ + Order, FundingOffer + + from pyee.base import EventEmitter class AuthEventsHandler: __once_abbreviations = { @@ -22,16 +31,6 @@ class AuthEventsHandler: **__on_abbreviations } - __serializers = { - ("os", "on", "ou", "oc",): serializers.Order, - ("ps", "pn", "pu", "pc",): serializers.Position, - ("te", "tu"): serializers.Trade, - ("fos", "fon", "fou", "foc",): serializers.FundingOffer, - ("fcs", "fcn", "fcu", "fcc",): serializers.FundingCredit, - ("fls", "fln", "flu", "flc",): serializers.FundingLoan, - ("ws", "wu",): serializers.Wallet - } - ONCE_EVENTS = [ *list(__once_abbreviations.values()) ] @@ -42,29 +41,44 @@ class AuthEventsHandler: "oc-req-notification", "fon-req-notification", "foc-req-notification" ] - def __init__(self, event_emitter): - self.event_emitter = event_emitter + def __init__(self, event_emitter: "EventEmitter") -> None: + self.__event_emitter = event_emitter - def handle(self, abbrevation, stream): + self.__serializers: Dict[Tuple[str, ...], serializers._Serializer] = { + ("os", "on", "ou", "oc",): serializers.Order, + ("ps", "pn", "pu", "pc",): serializers.Position, + ("te", "tu"): serializers.Trade, + ("fos", "fon", "fou", "foc",): serializers.FundingOffer, + ("fcs", "fcn", "fcu", "fcc",): serializers.FundingCredit, + ("fls", "fln", "flu", "flc",): serializers.FundingLoan, + ("ws", "wu",): serializers.Wallet + } + + def handle(self, abbrevation: str, stream: Any) -> None: if abbrevation == "n": return self.__notification(stream) - for abbrevations, serializer in AuthEventsHandler.__serializers.items(): + for abbrevations, serializer in self.__serializers.items(): if abbrevation in abbrevations: event = AuthEventsHandler.__abbreviations[abbrevation] - if all(isinstance(substream, list) for substream in stream): - return self.event_emitter.emit(event, [ serializer.parse(*substream) for substream in stream ]) + if all(isinstance(sub_stream, list) for sub_stream in stream): + data = [ serializer.parse(*sub_stream) for sub_stream in stream ] + else: data = serializer.parse(*stream) - return self.event_emitter.emit(event, serializer.parse(*stream)) + self.__event_emitter.emit(event, data) - def __notification(self, stream): - event, serializer = "notification", _Notification(serializer=None) + break + + def __notification(self, stream: Any) -> None: + _Types = Union[None, "Order", "FundingOffer"] + + event, serializer = "notification", _Notification[_Types](serializer=None) if stream[1] == "on-req" or stream[1] == "ou-req" or stream[1] == "oc-req": - event, serializer = f"{stream[1]}-notification", _Notification(serializer=serializers.Order) + event, serializer = f"{stream[1]}-notification", _Notification[_Types](serializer=serializers.Order) if stream[1] == "fon-req" or stream[1] == "foc-req": - event, serializer = f"{stream[1]}-notification", _Notification(serializer=serializers.FundingOffer) + event, serializer = f"{stream[1]}-notification", _Notification[_Types](serializer=serializers.FundingOffer) - return self.event_emitter.emit(event, serializer.parse(*stream)) + self.__event_emitter.emit(event, serializer.parse(*stream)) diff --git a/bfxapi/websocket/handlers/public_channels_handler.py b/bfxapi/websocket/handlers/public_channels_handler.py index f32fe14..456ecd8 100644 --- a/bfxapi/websocket/handlers/public_channels_handler.py +++ b/bfxapi/websocket/handlers/public_channels_handler.py @@ -1,4 +1,16 @@ -from ...types import serializers +from typing import TYPE_CHECKING, \ + Union, Dict, List, Any, cast + +from bfxapi.types import serializers + +if TYPE_CHECKING: + from bfxapi.websocket.subscriptions import Subscription, \ + Ticker, Trades, Book, Candles, Status + + from pyee.base import EventEmitter + + _NoHeaderSubscription = \ + Union[Ticker, Trades, Book, Candles, Status] class PublicChannelsHandler: ONCE_PER_SUBSCRIPTION_EVENTS = [ @@ -15,28 +27,32 @@ class PublicChannelsHandler: "f_raw_book_update", "candles_update", "derivatives_status_update" ] - def __init__(self, event_emitter, events_per_subscription): + def __init__(self, + event_emitter: "EventEmitter", + events_per_subscription: Dict[str, List[str]]) -> None: self.__event_emitter, self.__events_per_subscription = \ event_emitter, events_per_subscription - self.__handlers = { - "ticker": self.__ticker_channel_handler, - "trades": self.__trades_channel_handler, - "book": self.__book_channel_handler, - "candles": self.__candles_channel_handler, - "status": self.__status_channel_handler - } + def handle(self, subscription: "Subscription", stream: List[Any]) -> None: + def _strip(subscription: "Subscription", *args: str) -> "_NoHeaderSubscription": + return cast("_NoHeaderSubscription", \ + { key: value for key, value in subscription.items() if key not in args }) - def handle(self, subscription, *stream): - #pylint: disable-next=unnecessary-lambda-assignment - _clear = lambda dictionary, *args: { key: value for key, value in dictionary.items() if key not in args } + _subscription = _strip(subscription, "event", "channel", "chanId") - #pylint: disable-next=consider-iterating-dictionary - if (channel := subscription["channel"]) and channel in self.__handlers.keys(): - return self.__handlers[channel](_clear(subscription, "event", "channel", "chanId"), *stream) + if subscription["channel"] == "ticker": + self.__ticker_channel_handler(cast("Ticker", _subscription), stream) + elif subscription["channel"] == "trades": + self.__trades_channel_handler(cast("Trades", _subscription), stream) + elif subscription["channel"] == "book": + self.__book_channel_handler(cast("Book", _subscription), stream) + elif subscription["channel"] == "candles": + self.__candles_channel_handler(cast("Candles", _subscription), stream) + elif subscription["channel"] == "status": + self.__status_channel_handler(cast("Status", _subscription), stream) - def __emit(self, event, sub, data): - sub_id, should_emit_event = sub["subId"], True + def __emit(self, event: str, subscription: "_NoHeaderSubscription", data: Any) -> None: + sub_id, should_emit_event = subscription["subId"], True if event in PublicChannelsHandler.ONCE_PER_SUBSCRIPTION_EVENTS: if sub_id not in self.__events_per_subscription: @@ -46,94 +62,76 @@ class PublicChannelsHandler: else: should_emit_event = False if should_emit_event: - return self.__event_emitter.emit(event, sub, data) + self.__event_emitter.emit(event, subscription, data) - def __ticker_channel_handler(self, subscription, *stream): + def __ticker_channel_handler(self, subscription: "Ticker", stream: List[Any]) -> None: if subscription["symbol"].startswith("t"): - return self.__emit( - "t_ticker_update", - subscription, - serializers.TradingPairTicker.parse(*stream[0]) - ) + return self.__emit("t_ticker_update", subscription, \ + serializers.TradingPairTicker.parse(*stream[0])) if subscription["symbol"].startswith("f"): - return self.__emit( - "f_ticker_update", - subscription, - serializers.FundingCurrencyTicker.parse(*stream[0]) - ) + return self.__emit("f_ticker_update", subscription, \ + serializers.FundingCurrencyTicker.parse(*stream[0])) - def __trades_channel_handler(self, subscription, *stream): + def __trades_channel_handler(self, subscription: "Trades", stream: List[Any]) -> None: if (event := stream[0]) and event in [ "te", "tu", "fte", "ftu" ]: + events = { "te": "t_trade_execution", "tu": "t_trade_execution_update", \ + "fte": "f_trade_execution", "ftu": "f_trade_execution_update" } + if subscription["symbol"].startswith("t"): - return self.__emit( - { "te": "t_trade_execution", "tu": "t_trade_execution_update" }[event], - subscription, - serializers.TradingPairTrade.parse(*stream[1]) - ) + return self.__emit(events[event], subscription, \ + serializers.TradingPairTrade.parse(*stream[1])) if subscription["symbol"].startswith("f"): - return self.__emit( - { "fte": "f_trade_execution", "ftu": "f_trade_execution_update" }[event], - subscription, - serializers.FundingCurrencyTrade.parse(*stream[1]) - ) + return self.__emit(events[event], subscription, \ + serializers.FundingCurrencyTrade.parse(*stream[1])) if subscription["symbol"].startswith("t"): - return self.__emit( - "t_trades_snapshot", - subscription, - [ serializers.TradingPairTrade.parse(*substream) for substream in stream[0] ] - ) + return self.__emit("t_trades_snapshot", subscription, \ + [ serializers.TradingPairTrade.parse(*sub_stream) \ + for sub_stream in stream[0] ]) if subscription["symbol"].startswith("f"): - return self.__emit( - "f_trades_snapshot", - subscription, - [ serializers.FundingCurrencyTrade.parse(*substream) for substream in stream[0] ] - ) + return self.__emit("f_trades_snapshot", subscription, \ + [ serializers.FundingCurrencyTrade.parse(*sub_stream) \ + for sub_stream in stream[0] ]) - def __book_channel_handler(self, subscription, *stream): - event = subscription["symbol"][0] + def __book_channel_handler(self, subscription: "Book", stream: List[Any]) -> None: + t_or_f = subscription["symbol"][0] - if subscription["prec"] == "R0": - _trading_pair_serializer, _funding_currency_serializer, is_raw_book = \ - serializers.TradingPairRawBook, serializers.FundingCurrencyRawBook, True - else: _trading_pair_serializer, _funding_currency_serializer, is_raw_book = \ - serializers.TradingPairBook, serializers.FundingCurrencyBook, False + is_raw_book = subscription["prec"] == "R0" - if all(isinstance(substream, list) for substream in stream[0]): - return self.__emit( - event + "_" + (is_raw_book and "raw_book" or "book") + "_snapshot", - subscription, - [ { "t": _trading_pair_serializer, "f": _funding_currency_serializer }[event] \ - .parse(*substream) for substream in stream[0] ] - ) + serializer = { + "t": is_raw_book and serializers.TradingPairRawBook \ + or serializers.TradingPairBook, + "f": is_raw_book and serializers.FundingCurrencyRawBook \ + or serializers.FundingCurrencyBook + }[t_or_f] - return self.__emit( - event + "_" + (is_raw_book and "raw_book" or "book") + "_update", - subscription, - { "t": _trading_pair_serializer, "f": _funding_currency_serializer }[event].parse(*stream[0]) - ) + if all(isinstance(sub_stream, list) for sub_stream in stream[0]): + event = t_or_f + "_" + \ + (is_raw_book and "raw_book" or "book") + "_snapshot" - def __candles_channel_handler(self, subscription, *stream): - if all(isinstance(substream, list) for substream in stream[0]): - return self.__emit( - "candles_snapshot", - subscription, - [ serializers.Candle.parse(*substream) for substream in stream[0] ] - ) + return self.__emit(event, subscription, \ + [ serializer.parse(*sub_stream) \ + for sub_stream in stream[0] ]) - return self.__emit( - "candles_update", - subscription, - serializers.Candle.parse(*stream[0]) - ) + event = t_or_f + "_" + \ + (is_raw_book and "raw_book" or "book") + "_update" - def __status_channel_handler(self, subscription, *stream): + return self.__emit(event, subscription, \ + serializer.parse(*stream[0])) + + def __candles_channel_handler(self, subscription: "Candles", stream: List[Any]) -> None: + if all(isinstance(sub_stream, list) for sub_stream in stream[0]): + return self.__emit("candles_snapshot", subscription, \ + [ serializers.Candle.parse(*sub_stream) \ + for sub_stream in stream[0] ]) + + return self.__emit("candles_update", subscription, \ + serializers.Candle.parse(*stream[0])) + + def __status_channel_handler(self, subscription: "Status", stream: List[Any]) -> None: if subscription["key"].startswith("deriv:"): - return self.__emit( - "derivatives_status_update", - subscription, - serializers.DerivativesStatus.parse(*stream[0]) - ) + return self.__emit("derivatives_status_update", subscription, \ + serializers.DerivativesStatus.parse(*stream[0])) diff --git a/bfxapi/websocket/subscriptions.py b/bfxapi/websocket/subscriptions.py index 72df007..d4ffd2a 100644 --- a/bfxapi/websocket/subscriptions.py +++ b/bfxapi/websocket/subscriptions.py @@ -1,20 +1,5 @@ -from typing import TypedDict, Union, Literal, Optional - -_Channel = Literal[ - "ticker", - "trades", - "book", - "candles", - "status" -] - -_Header = TypedDict("_Header", { - "event": Literal["subscribed"], - "channel": _Channel, - "chanId": int -}) - -Subscription = Union[_Header, "Ticker", "Trades", "Book", "Candles", "Status"] +from typing import TypedDict, \ + Union, Literal, Optional class Ticker(TypedDict): subId: str @@ -43,3 +28,33 @@ class Candles(TypedDict): class Status(TypedDict): subId: str key: str + +Subscription = Union["_Ticker", "_Trades", "_Book", "_Candles", "_Status"] + +_Channel = Literal["ticker", "trades", "book", "candles", "status"] + +_Header = TypedDict("_Header", { + "event": Literal["subscribed"], + "channel": _Channel, + "chanId": int +}) + +#pylint: disable-next=inherit-non-class +class _Ticker(Ticker, _Header): + pass + +#pylint: disable-next=inherit-non-class +class _Trades(Trades, _Header): + pass + +#pylint: disable-next=inherit-non-class +class _Book(Book, _Header): + pass + +#pylint: disable-next=inherit-non-class +class _Candles(Candles, _Header): + pass + +#pylint: disable-next=inherit-non-class +class _Status(Status, _Header): + pass diff --git a/dev-requirements.txt b/dev-requirements.txt index fff03cf052a26593c67097d8c3c4cd6ce90c4e5b..c7daa7862d8b787e838ecbbee8fcad38190de71d 100644 GIT binary patch delta 20 acmcb?a)V`p4W}W49)mFu8*FqHW&!{? Date: Thu, 25 May 2023 20:40:31 +0200 Subject: [PATCH 04/65] Add type hints and type checks in bfxapi.websocket.client.bfx_websocket_inputs. --- .../websocket/client/bfx_websocket_inputs.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/bfxapi/websocket/client/bfx_websocket_inputs.py b/bfxapi/websocket/client/bfx_websocket_inputs.py index 960f886..46a3de8 100644 --- a/bfxapi/websocket/client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/client/bfx_websocket_inputs.py @@ -1,16 +1,24 @@ +from typing import TYPE_CHECKING, Callable, Awaitable, \ + Tuple, List, Union, Optional, Any + from decimal import Decimal + from datetime import datetime -from typing import Union, Optional, List, Tuple -from ..enums import OrderType, FundingOfferType -from ...types import JSON +if TYPE_CHECKING: + from bfxapi.enums import \ + OrderType, FundingOfferType + + from bfxapi.types import JSON + + _Handler = Callable[[str, Any], Awaitable[None]] class BfxWebSocketInputs: - def __init__(self, handle_websocket_input): + def __init__(self, handle_websocket_input: "_Handler") -> None: self.__handle_websocket_input = handle_websocket_input async def submit_order(self, - type: OrderType, + type: "OrderType", symbol: str, amount: Union[Decimal, float, str], *, @@ -23,7 +31,7 @@ class BfxWebSocketInputs: cid: Optional[int] = None, flags: Optional[int] = 0, tif: Optional[Union[datetime, str]] = None, - meta: Optional[JSON] = None): + meta: Optional["JSON"] = None) -> None: await self.__handle_websocket_input("on", { "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, @@ -45,7 +53,7 @@ class BfxWebSocketInputs: delta: Optional[Union[Decimal, float, str]] = None, price_aux_limit: Optional[Union[Decimal, float, str]] = None, price_trailing: Optional[Union[Decimal, float, str]] = None, - tif: Optional[Union[datetime, str]] = None): + tif: Optional[Union[datetime, str]] = None) -> None: await self.__handle_websocket_input("ou", { "id": id, "amount": amount, "price": price, "cid": cid, "cid_date": cid_date, "gid": gid, @@ -57,7 +65,7 @@ class BfxWebSocketInputs: *, id: Optional[int] = None, cid: Optional[int] = None, - cid_date: Optional[str] = None): + cid_date: Optional[str] = None) -> None: await self.__handle_websocket_input("oc", { "id": id, "cid": cid, "cid_date": cid_date }) @@ -67,7 +75,7 @@ class BfxWebSocketInputs: ids: Optional[List[int]] = None, cids: Optional[List[Tuple[int, str]]] = None, gids: Optional[List[int]] = None, - all: bool = False): + all: bool = False) -> None: await self.__handle_websocket_input("oc_multi", { "ids": ids, "cids": cids, "gids": gids, "all": int(all) @@ -75,20 +83,20 @@ class BfxWebSocketInputs: #pylint: disable-next=too-many-arguments async def submit_funding_offer(self, - type: FundingOfferType, + type: "FundingOfferType", symbol: str, amount: Union[Decimal, float, str], rate: Union[Decimal, float, str], period: int, *, - flags: Optional[int] = 0): + flags: Optional[int] = 0) -> None: await self.__handle_websocket_input("fon", { "type": type, "symbol": symbol, "amount": amount, "rate": rate, "period": period, "flags": flags }) - async def cancel_funding_offer(self, id: int): + async def cancel_funding_offer(self, id: int) -> None: await self.__handle_websocket_input("foc", { "id": id }) - async def calc(self, *args: str): + async def calc(self, *args: str) -> None: await self.__handle_websocket_input("calc", list(map(lambda arg: [arg], args))) From bc0f83d408cd3c8a37071f8d3c84717fe71077c2 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 26 May 2023 18:02:41 +0200 Subject: [PATCH 05/65] Improve JSONEncoder class in bfxapi.utils.json_encoder. --- .../endpoints/rest_authenticated_endpoints.py | 6 +- bfxapi/utils/json_encoder.py | 10 +- .../websocket/client/bfx_websocket_inputs.py | 99 +++++++++---------- 3 files changed, 58 insertions(+), 57 deletions(-) diff --git a/bfxapi/rest/endpoints/rest_authenticated_endpoints.py b/bfxapi/rest/endpoints/rest_authenticated_endpoints.py index b9e9ff3..db1c37e 100644 --- a/bfxapi/rest/endpoints/rest_authenticated_endpoints.py +++ b/bfxapi/rest/endpoints/rest_authenticated_endpoints.py @@ -128,7 +128,7 @@ class RestAuthenticatedEndpoints(Middleware): all: bool = False) -> Notification[List[Order]]: body = { "ids": ids, "cids": cids, "gids": gids, - "all": int(all) + "all": all } return _Notification[List[Order]](serializers.Order, is_iterable=True) \ @@ -319,7 +319,7 @@ class RestAuthenticatedEndpoints(Middleware): rate: Optional[int] = None, period: Optional[int] = None) -> Notification[FundingAutoRenew]: body = { - "status": int(status), "currency": currency, "amount": amount, + "status": status, "currency": currency, "amount": amount, "rate": rate, "period": period } @@ -449,7 +449,7 @@ class RestAuthenticatedEndpoints(Middleware): renew: bool = False) -> Notification[DepositAddress]: return _Notification[DepositAddress](serializers.DepositAddress) \ .parse(*self._post("auth/w/deposit/address", \ - body={ "wallet": wallet, "method": method, "renew": int(renew) })) + body={ "wallet": wallet, "method": method, "renew": renew })) def generate_deposit_invoice(self, wallet: str, diff --git a/bfxapi/utils/json_encoder.py b/bfxapi/utils/json_encoder.py index 21f0b7e..3da4c99 100644 --- a/bfxapi/utils/json_encoder.py +++ b/bfxapi/utils/json_encoder.py @@ -9,18 +9,20 @@ JSON = Union[Dict[str, "JSON"], List["JSON"], bool, int, float, str, Type[None]] def _strip(dictionary: Dict) -> Dict: return { key: value for key, value in dictionary.items() if value is not None } -def _convert_float_to_str(data: JSON) -> JSON: +def _convert_data_to_json(data: JSON) -> JSON: + if isinstance(data, bool): + return int(data) if isinstance(data, float): return format(Decimal(repr(data)), "f") if isinstance(data, list): - return [ _convert_float_to_str(sub_data) for sub_data in data ] + return [ _convert_data_to_json(sub_data) for sub_data in data ] if isinstance(data, dict): - return _strip({ key: _convert_float_to_str(value) for key, value in data.items() }) + return _strip({ key: _convert_data_to_json(value) for key, value in data.items() }) return data class JSONEncoder(json.JSONEncoder): def encode(self, o: JSON) -> str: - return json.JSONEncoder.encode(self, _convert_float_to_str(o)) + return json.JSONEncoder.encode(self, _convert_data_to_json(o)) def default(self, o: Any) -> Any: if isinstance(o, Decimal): diff --git a/bfxapi/websocket/client/bfx_websocket_inputs.py b/bfxapi/websocket/client/bfx_websocket_inputs.py index 46a3de8..0725f15 100644 --- a/bfxapi/websocket/client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/client/bfx_websocket_inputs.py @@ -1,37 +1,36 @@ from typing import TYPE_CHECKING, Callable, Awaitable, \ Tuple, List, Union, Optional, Any -from decimal import Decimal - -from datetime import datetime - if TYPE_CHECKING: from bfxapi.enums import \ OrderType, FundingOfferType from bfxapi.types import JSON - _Handler = Callable[[str, Any], Awaitable[None]] + from decimal import Decimal + + from datetime import datetime class BfxWebSocketInputs: - def __init__(self, handle_websocket_input: "_Handler") -> None: + def __init__(self, handle_websocket_input: Callable[[str, Any], Awaitable[None]]) -> None: self.__handle_websocket_input = handle_websocket_input async def submit_order(self, - type: "OrderType", - symbol: str, - amount: Union[Decimal, float, str], - *, - price: Optional[Union[Decimal, float, str]] = None, - lev: Optional[int] = None, - price_trailing: Optional[Union[Decimal, float, str]] = None, - price_aux_limit: Optional[Union[Decimal, float, str]] = None, - price_oco_stop: Optional[Union[Decimal, float, str]] = None, - gid: Optional[int] = None, - cid: Optional[int] = None, - flags: Optional[int] = 0, - tif: Optional[Union[datetime, str]] = None, - meta: Optional["JSON"] = None) -> None: + type: "OrderType", + symbol: str, + amount: Union["Decimal", float, str], + *, + price: Optional[Union["Decimal", float, str]] = None, + lev: Optional[int] = None, + price_trailing: Optional[Union["Decimal", float, str]] = None, + price_aux_limit: Optional[Union["Decimal", float, str]] = None, + price_oco_stop: Optional[Union["Decimal", float, str]] = None, + gid: Optional[int] = None, + cid: Optional[int] = None, + flags: Optional[int] = 0, + tif: Optional[Union["datetime", str]] = None, + meta: Optional["JSON"] = None) -> None: + await self.__handle_websocket_input("on", { "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, @@ -41,19 +40,19 @@ class BfxWebSocketInputs: }) async def update_order(self, - id: int, - *, - amount: Optional[Union[Decimal, float, str]] = None, - price: Optional[Union[Decimal, float, str]] = None, - cid: Optional[int] = None, - cid_date: Optional[str] = None, - gid: Optional[int] = None, - flags: Optional[int] = 0, - lev: Optional[int] = None, - delta: Optional[Union[Decimal, float, str]] = None, - price_aux_limit: Optional[Union[Decimal, float, str]] = None, - price_trailing: Optional[Union[Decimal, float, str]] = None, - tif: Optional[Union[datetime, str]] = None) -> None: + id: int, + *, + amount: Optional[Union["Decimal", float, str]] = None, + price: Optional[Union["Decimal", float, str]] = None, + cid: Optional[int] = None, + cid_date: Optional[str] = None, + gid: Optional[int] = None, + flags: Optional[int] = 0, + lev: Optional[int] = None, + delta: Optional[Union["Decimal", float, str]] = None, + price_aux_limit: Optional[Union["Decimal", float, str]] = None, + price_trailing: Optional[Union["Decimal", float, str]] = None, + tif: Optional[Union["datetime", str]] = None) -> None: await self.__handle_websocket_input("ou", { "id": id, "amount": amount, "price": price, "cid": cid, "cid_date": cid_date, "gid": gid, @@ -62,34 +61,34 @@ class BfxWebSocketInputs: }) async def cancel_order(self, - *, - id: Optional[int] = None, - cid: Optional[int] = None, - cid_date: Optional[str] = None) -> None: + *, + id: Optional[int] = None, + cid: Optional[int] = None, + cid_date: Optional[str] = None) -> None: await self.__handle_websocket_input("oc", { "id": id, "cid": cid, "cid_date": cid_date }) async def cancel_order_multi(self, - *, - ids: Optional[List[int]] = None, - cids: Optional[List[Tuple[int, str]]] = None, - gids: Optional[List[int]] = None, - all: bool = False) -> None: + *, + ids: Optional[List[int]] = None, + cids: Optional[List[Tuple[int, str]]] = None, + gids: Optional[List[int]] = None, + all: bool = False) -> None: await self.__handle_websocket_input("oc_multi", { "ids": ids, "cids": cids, "gids": gids, - "all": int(all) + "all": all }) #pylint: disable-next=too-many-arguments async def submit_funding_offer(self, - type: "FundingOfferType", - symbol: str, - amount: Union[Decimal, float, str], - rate: Union[Decimal, float, str], - period: int, - *, - flags: Optional[int] = 0) -> None: + type: "FundingOfferType", + symbol: str, + amount: Union["Decimal", float, str], + rate: Union["Decimal", float, str], + period: int, + *, + flags: Optional[int] = 0) -> None: await self.__handle_websocket_input("fon", { "type": type, "symbol": symbol, "amount": amount, "rate": rate, "period": period, "flags": flags From 7059846843e3ebdc04b7a6f8c6249128c3942bf4 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 26 May 2023 18:48:27 +0200 Subject: [PATCH 06/65] Remove support for datetime type and improve typing in several files. --- .../endpoints/rest_authenticated_endpoints.py | 10 +++++--- bfxapi/types/__init__.py | 2 +- bfxapi/types/dataclasses.py | 4 +-- bfxapi/utils/json_encoder.py | 25 +++++++++---------- .../websocket/client/bfx_websocket_inputs.py | 9 +++---- 5 files changed, 24 insertions(+), 26 deletions(-) diff --git a/bfxapi/rest/endpoints/rest_authenticated_endpoints.py b/bfxapi/rest/endpoints/rest_authenticated_endpoints.py index db1c37e..acb4757 100644 --- a/bfxapi/rest/endpoints/rest_authenticated_endpoints.py +++ b/bfxapi/rest/endpoints/rest_authenticated_endpoints.py @@ -1,12 +1,12 @@ from typing import Dict, List, Tuple, Union, Literal, Optional + from decimal import Decimal -from datetime import datetime from ..middleware import Middleware from ..enums import Sort, OrderType, FundingOfferType -from ...types import JSON, Notification, \ +from ...types import Notification, \ UserInfo, LoginHistory, BalanceAvailable, \ Order, Position, Trade, \ FundingTrade, OrderTrade, Ledger, \ @@ -22,6 +22,8 @@ from ...types import serializers from ...types.serializers import _Notification +from ...utils.json_encoder import JSON + class RestAuthenticatedEndpoints(Middleware): def get_user_info(self) -> UserInfo: return serializers.UserInfo \ @@ -74,7 +76,7 @@ class RestAuthenticatedEndpoints(Middleware): gid: Optional[int] = None, cid: Optional[int] = None, flags: Optional[int] = 0, - tif: Optional[Union[datetime, str]] = None, + tif: Optional[str] = None, meta: Optional[JSON] = None) -> Notification[Order]: body = { "type": type, "symbol": symbol, "amount": amount, @@ -100,7 +102,7 @@ class RestAuthenticatedEndpoints(Middleware): delta: Optional[Union[Decimal, float, str]] = None, price_aux_limit: Optional[Union[Decimal, float, str]] = None, price_trailing: Optional[Union[Decimal, float, str]] = None, - tif: Optional[Union[datetime, str]] = None) -> Notification[Order]: + tif: Optional[str] = None) -> Notification[Order]: body = { "id": id, "amount": amount, "price": price, "cid": cid, "cid_date": cid_date, "gid": gid, diff --git a/bfxapi/types/__init__.py b/bfxapi/types/__init__.py index ce3ef06..2648d38 100644 --- a/bfxapi/types/__init__.py +++ b/bfxapi/types/__init__.py @@ -1,4 +1,4 @@ -from .dataclasses import JSON, \ +from .dataclasses import \ PlatformStatus, TradingPairTicker, FundingCurrencyTicker, \ TickersHistory, TradingPairTrade, FundingCurrencyTrade, \ TradingPairBook, FundingCurrencyBook, TradingPairRawBook, \ diff --git a/bfxapi/types/dataclasses.py b/bfxapi/types/dataclasses.py index 264de42..4c375e5 100644 --- a/bfxapi/types/dataclasses.py +++ b/bfxapi/types/dataclasses.py @@ -1,11 +1,11 @@ -from typing import Union, Type, \ +from typing import \ List, Dict, Literal, Optional, Any from dataclasses import dataclass from .labeler import _Type, partial, compose -JSON = Union[Dict[str, "JSON"], List["JSON"], bool, int, float, str, Type[None]] +from ..utils.json_encoder import JSON #region Dataclass definitions for types of public use diff --git a/bfxapi/utils/json_encoder.py b/bfxapi/utils/json_encoder.py index 3da4c99..9807480 100644 --- a/bfxapi/utils/json_encoder.py +++ b/bfxapi/utils/json_encoder.py @@ -1,33 +1,32 @@ import json + from decimal import Decimal -from datetime import datetime -from typing import Type, List, Dict, Union, Any +from typing import List, Dict, Union -JSON = Union[Dict[str, "JSON"], List["JSON"], bool, int, float, str, Type[None]] +JSON = Union[Dict[str, "JSON"], List["JSON"], bool, int, float, str, None] + +_CustomJSON = Union[Dict[str, "_CustomJSON"], List["_CustomJSON"], \ + bool, int, float, str, Decimal, None] def _strip(dictionary: Dict) -> Dict: return { key: value for key, value in dictionary.items() if value is not None } -def _convert_data_to_json(data: JSON) -> JSON: +def _convert_data_to_json(data: _CustomJSON) -> JSON: if isinstance(data, bool): return int(data) if isinstance(data, float): return format(Decimal(repr(data)), "f") + if isinstance(data, Decimal): + return format(data, "f") + if isinstance(data, list): return [ _convert_data_to_json(sub_data) for sub_data in data ] if isinstance(data, dict): return _strip({ key: _convert_data_to_json(value) for key, value in data.items() }) + return data class JSONEncoder(json.JSONEncoder): - def encode(self, o: JSON) -> str: + def encode(self, o: _CustomJSON) -> str: return json.JSONEncoder.encode(self, _convert_data_to_json(o)) - - def default(self, o: Any) -> Any: - if isinstance(o, Decimal): - return format(o, "f") - if isinstance(o, datetime): - return str(o) - - return json.JSONEncoder.default(self, o) diff --git a/bfxapi/websocket/client/bfx_websocket_inputs.py b/bfxapi/websocket/client/bfx_websocket_inputs.py index 0725f15..87d730b 100644 --- a/bfxapi/websocket/client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/client/bfx_websocket_inputs.py @@ -5,12 +5,10 @@ if TYPE_CHECKING: from bfxapi.enums import \ OrderType, FundingOfferType - from bfxapi.types import JSON + from bfxapi.utils.json_encoder import JSON from decimal import Decimal - from datetime import datetime - class BfxWebSocketInputs: def __init__(self, handle_websocket_input: Callable[[str, Any], Awaitable[None]]) -> None: self.__handle_websocket_input = handle_websocket_input @@ -28,9 +26,8 @@ class BfxWebSocketInputs: gid: Optional[int] = None, cid: Optional[int] = None, flags: Optional[int] = 0, - tif: Optional[Union["datetime", str]] = None, + tif: Optional[str] = None, meta: Optional["JSON"] = None) -> None: - await self.__handle_websocket_input("on", { "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, @@ -52,7 +49,7 @@ class BfxWebSocketInputs: delta: Optional[Union["Decimal", float, str]] = None, price_aux_limit: Optional[Union["Decimal", float, str]] = None, price_trailing: Optional[Union["Decimal", float, str]] = None, - tif: Optional[Union["datetime", str]] = None) -> None: + tif: Optional[str] = None) -> None: await self.__handle_websocket_input("ou", { "id": id, "amount": amount, "price": price, "cid": cid, "cid_date": cid_date, "gid": gid, From 708fdc87c78f7f803baa7e7955eb132f430a1f49 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 12 Jun 2023 15:37:05 +0200 Subject: [PATCH 07/65] Add new event liquidation_feed_update to PublicChannelsHandler (and improve overall type hinting). --- bfxapi/types/dataclasses.py | 2 +- bfxapi/types/serializers.py | 2 +- bfxapi/websocket/enums.py | 2 +- bfxapi/websocket/handlers/auth_events_handler.py | 12 +++++++----- .../websocket/handlers/public_channels_handler.py | 15 ++++++++++----- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/bfxapi/types/dataclasses.py b/bfxapi/types/dataclasses.py index 4c375e5..8b0b69f 100644 --- a/bfxapi/types/dataclasses.py +++ b/bfxapi/types/dataclasses.py @@ -129,7 +129,7 @@ class Liquidation(_Type): base_price: float is_match: int is_market_sold: int - price_acquired: float + liquidation_price: float @dataclass class Leaderboard(_Type): diff --git a/bfxapi/types/serializers.py b/bfxapi/types/serializers.py index f7838bf..d1f2ab9 100644 --- a/bfxapi/types/serializers.py +++ b/bfxapi/types/serializers.py @@ -230,7 +230,7 @@ Liquidation = generate_labeler_serializer( "is_match", "is_market_sold", "_PLACEHOLDER", - "price_acquired" + "liquidation_price" ] ) diff --git a/bfxapi/websocket/enums.py b/bfxapi/websocket/enums.py index 227bf69..8fe6028 100644 --- a/bfxapi/websocket/enums.py +++ b/bfxapi/websocket/enums.py @@ -1,5 +1,5 @@ #pylint: disable-next=wildcard-import,unused-wildcard-import -from ..enums import * +from bfxapi.enums import * class Channel(str, Enum): TICKER = "ticker" diff --git a/bfxapi/websocket/handlers/auth_events_handler.py b/bfxapi/websocket/handlers/auth_events_handler.py index 8c5cab3..f411e7b 100644 --- a/bfxapi/websocket/handlers/auth_events_handler.py +++ b/bfxapi/websocket/handlers/auth_events_handler.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING, \ - Union, Dict, Tuple, Any + Dict, Tuple, Any from bfxapi.types import serializers @@ -71,14 +71,16 @@ class AuthEventsHandler: break def __notification(self, stream: Any) -> None: - _Types = Union[None, "Order", "FundingOffer"] + event: str = "notification" - event, serializer = "notification", _Notification[_Types](serializer=None) + serializer: _Notification = _Notification[None](serializer=None) if stream[1] == "on-req" or stream[1] == "ou-req" or stream[1] == "oc-req": - event, serializer = f"{stream[1]}-notification", _Notification[_Types](serializer=serializers.Order) + event, serializer = f"{stream[1]}-notification", \ + _Notification["Order"](serializer=serializers.Order) if stream[1] == "fon-req" or stream[1] == "foc-req": - event, serializer = f"{stream[1]}-notification", _Notification[_Types](serializer=serializers.FundingOffer) + event, serializer = f"{stream[1]}-notification", \ + _Notification["FundingOffer"](serializer=serializers.FundingOffer) self.__event_emitter.emit(event, serializer.parse(*stream)) diff --git a/bfxapi/websocket/handlers/public_channels_handler.py b/bfxapi/websocket/handlers/public_channels_handler.py index 456ecd8..6a4a919 100644 --- a/bfxapi/websocket/handlers/public_channels_handler.py +++ b/bfxapi/websocket/handlers/public_channels_handler.py @@ -24,7 +24,8 @@ class PublicChannelsHandler: "t_ticker_update", "f_ticker_update", "t_trade_execution", "t_trade_execution_update", "f_trade_execution", "f_trade_execution_update", "t_book_update", "f_book_update", "t_raw_book_update", - "f_raw_book_update", "candles_update", "derivatives_status_update" + "f_raw_book_update", "candles_update", "derivatives_status_update", + "liquidation_feed_update" ] def __init__(self, @@ -97,7 +98,7 @@ class PublicChannelsHandler: for sub_stream in stream[0] ]) def __book_channel_handler(self, subscription: "Book", stream: List[Any]) -> None: - t_or_f = subscription["symbol"][0] + symbol = subscription["symbol"] is_raw_book = subscription["prec"] == "R0" @@ -106,17 +107,17 @@ class PublicChannelsHandler: or serializers.TradingPairBook, "f": is_raw_book and serializers.FundingCurrencyRawBook \ or serializers.FundingCurrencyBook - }[t_or_f] + }[symbol[0]] if all(isinstance(sub_stream, list) for sub_stream in stream[0]): - event = t_or_f + "_" + \ + event = symbol[0] + "_" + \ (is_raw_book and "raw_book" or "book") + "_snapshot" return self.__emit(event, subscription, \ [ serializer.parse(*sub_stream) \ for sub_stream in stream[0] ]) - event = t_or_f + "_" + \ + event = symbol[0] + "_" + \ (is_raw_book and "raw_book" or "book") + "_update" return self.__emit(event, subscription, \ @@ -135,3 +136,7 @@ class PublicChannelsHandler: if subscription["key"].startswith("deriv:"): return self.__emit("derivatives_status_update", subscription, \ serializers.DerivativesStatus.parse(*stream[0])) + + if subscription["key"].startswith("liq:"): + return self.__emit("liquidation_feed_update", subscription, \ + serializers.Liquidation.parse(*stream[0][0])) From f343fce20f8465fd40786a6bb43f550f330b895f Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 12 Jun 2023 15:42:52 +0200 Subject: [PATCH 08/65] Fix comment on top of examples in both examples.rest.auth and examples.websocket.auth. --- examples/rest/auth/claim_position.py | 2 +- examples/rest/auth/get_wallets.py | 2 +- examples/rest/auth/set_derivative_position_collateral.py | 2 +- examples/rest/auth/submit_funding_offer.py | 2 +- examples/rest/auth/submit_order.py | 2 +- examples/rest/auth/toggle_keep_funding.py | 2 +- examples/websocket/auth/submit_order.py | 2 +- examples/websocket/auth/wallets.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/rest/auth/claim_position.py b/examples/rest/auth/claim_position.py index 53dfdb7..8243600 100644 --- a/examples/rest/auth/claim_position.py +++ b/examples/rest/auth/claim_position.py @@ -1,4 +1,4 @@ -# python -c "import examples.rest.authenticated.claim_position" +# python -c "import examples.rest.auth.claim_position" import os diff --git a/examples/rest/auth/get_wallets.py b/examples/rest/auth/get_wallets.py index effa431..b0c5aae 100644 --- a/examples/rest/auth/get_wallets.py +++ b/examples/rest/auth/get_wallets.py @@ -1,4 +1,4 @@ -# python -c "import examples.rest.authenticated.get_wallets" +# python -c "import examples.rest.auth.get_wallets" import os diff --git a/examples/rest/auth/set_derivative_position_collateral.py b/examples/rest/auth/set_derivative_position_collateral.py index 5097898..7365b83 100644 --- a/examples/rest/auth/set_derivative_position_collateral.py +++ b/examples/rest/auth/set_derivative_position_collateral.py @@ -1,4 +1,4 @@ -# python -c "import examples.rest.authenticated.set_derivatives_position_collateral" +# python -c "import examples.rest.auth.set_derivatives_position_collateral" import os diff --git a/examples/rest/auth/submit_funding_offer.py b/examples/rest/auth/submit_funding_offer.py index 2016fbe..2a29b27 100644 --- a/examples/rest/auth/submit_funding_offer.py +++ b/examples/rest/auth/submit_funding_offer.py @@ -1,4 +1,4 @@ -# python -c "import examples.rest.authenticated.submit_funding_offer" +# python -c "import examples.rest.auth.submit_funding_offer" import os diff --git a/examples/rest/auth/submit_order.py b/examples/rest/auth/submit_order.py index 4179ee9..8fc2830 100644 --- a/examples/rest/auth/submit_order.py +++ b/examples/rest/auth/submit_order.py @@ -1,4 +1,4 @@ -# python -c "import examples.rest.authenticated.submit_order" +# python -c "import examples.rest.auth.submit_order" import os diff --git a/examples/rest/auth/toggle_keep_funding.py b/examples/rest/auth/toggle_keep_funding.py index e1fbb78..8a1880d 100644 --- a/examples/rest/auth/toggle_keep_funding.py +++ b/examples/rest/auth/toggle_keep_funding.py @@ -1,4 +1,4 @@ -# python -c "import examples.rest.authenticated.toggle_keep_funding" +# python -c "import examples.rest.auth.toggle_keep_funding" import os diff --git a/examples/websocket/auth/submit_order.py b/examples/websocket/auth/submit_order.py index 4e5b8d6..0c2d03b 100644 --- a/examples/websocket/auth/submit_order.py +++ b/examples/websocket/auth/submit_order.py @@ -1,4 +1,4 @@ -# python -c "import examples.websocket.authenticated.submit_order" +# python -c "import examples.websocket.auth.submit_order" import os diff --git a/examples/websocket/auth/wallets.py b/examples/websocket/auth/wallets.py index 1773a3a..057ad29 100644 --- a/examples/websocket/auth/wallets.py +++ b/examples/websocket/auth/wallets.py @@ -1,4 +1,4 @@ -# python -c "import examples.websocket.authenticated.wallets" +# python -c "import examples.websocket.auth.wallets" import os From d63c2c63c3c45fa2690fe25ebd4bb4fc18ea90be Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 12 Jun 2023 15:45:28 +0200 Subject: [PATCH 09/65] Rename RestAuthenticatedEndpoints to RestAuthEndpoints (and bfxapi.rest.endpoints.rest_authenticated_endpoints to rest_auth_endpoints). --- bfxapi/rest/__init__.py | 2 +- bfxapi/rest/endpoints/__init__.py | 2 +- bfxapi/rest/endpoints/bfx_rest_interface.py | 4 ++-- ...rest_authenticated_endpoints.py => rest_auth_endpoints.py} | 2 +- bfxapi/websocket/client/bfx_websocket_inputs.py | 3 ++- 5 files changed, 7 insertions(+), 6 deletions(-) rename bfxapi/rest/endpoints/{rest_authenticated_endpoints.py => rest_auth_endpoints.py} (99%) diff --git a/bfxapi/rest/__init__.py b/bfxapi/rest/__init__.py index e18526e..aeff248 100644 --- a/bfxapi/rest/__init__.py +++ b/bfxapi/rest/__init__.py @@ -1,2 +1,2 @@ -from .endpoints import BfxRestInterface, RestPublicEndpoints, RestAuthenticatedEndpoints, \ +from .endpoints import BfxRestInterface, RestPublicEndpoints, RestAuthEndpoints, \ RestMerchantEndpoints diff --git a/bfxapi/rest/endpoints/__init__.py b/bfxapi/rest/endpoints/__init__.py index 2775e2e..96822c9 100644 --- a/bfxapi/rest/endpoints/__init__.py +++ b/bfxapi/rest/endpoints/__init__.py @@ -1,5 +1,5 @@ from .bfx_rest_interface import BfxRestInterface from .rest_public_endpoints import RestPublicEndpoints -from .rest_authenticated_endpoints import RestAuthenticatedEndpoints +from .rest_auth_endpoints import RestAuthEndpoints from .rest_merchant_endpoints import RestMerchantEndpoints diff --git a/bfxapi/rest/endpoints/bfx_rest_interface.py b/bfxapi/rest/endpoints/bfx_rest_interface.py index 12a06f4..73ec603 100644 --- a/bfxapi/rest/endpoints/bfx_rest_interface.py +++ b/bfxapi/rest/endpoints/bfx_rest_interface.py @@ -1,5 +1,5 @@ from .rest_public_endpoints import RestPublicEndpoints -from .rest_authenticated_endpoints import RestAuthenticatedEndpoints +from .rest_auth_endpoints import RestAuthEndpoints from .rest_merchant_endpoints import RestMerchantEndpoints class BfxRestInterface: @@ -9,5 +9,5 @@ class BfxRestInterface: api_key, api_secret = (credentials['api_key'], credentials['api_secret']) if credentials else (None, None) self.public = RestPublicEndpoints(host=host) - self.auth = RestAuthenticatedEndpoints(host=host, api_key=api_key, api_secret=api_secret) + self.auth = RestAuthEndpoints(host=host, api_key=api_key, api_secret=api_secret) self.merchant = RestMerchantEndpoints(host=host, api_key=api_key, api_secret=api_secret) diff --git a/bfxapi/rest/endpoints/rest_authenticated_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py similarity index 99% rename from bfxapi/rest/endpoints/rest_authenticated_endpoints.py rename to bfxapi/rest/endpoints/rest_auth_endpoints.py index acb4757..e1b9fab 100644 --- a/bfxapi/rest/endpoints/rest_authenticated_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -24,7 +24,7 @@ from ...types.serializers import _Notification from ...utils.json_encoder import JSON -class RestAuthenticatedEndpoints(Middleware): +class RestAuthEndpoints(Middleware): def get_user_info(self) -> UserInfo: return serializers.UserInfo \ .parse(*self._post("auth/r/info/user")) diff --git a/bfxapi/websocket/client/bfx_websocket_inputs.py b/bfxapi/websocket/client/bfx_websocket_inputs.py index 87d730b..71bf8a8 100644 --- a/bfxapi/websocket/client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/client/bfx_websocket_inputs.py @@ -95,4 +95,5 @@ class BfxWebSocketInputs: await self.__handle_websocket_input("foc", { "id": id }) async def calc(self, *args: str) -> None: - await self.__handle_websocket_input("calc", list(map(lambda arg: [arg], args))) + await self.__handle_websocket_input("calc", + list(map(lambda arg: [arg], args))) From cc5f9f5b0e9dbc42a494fa9fe01a220232786ca7 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 12 Jun 2023 16:17:45 +0200 Subject: [PATCH 10/65] Remove type hinting for decorators _require_websocket_connection and _require_websocket_authentication. --- bfxapi/websocket/client/bfx_websocket_bucket.py | 8 ++------ bfxapi/websocket/client/bfx_websocket_client.py | 8 +++----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/bfxapi/websocket/client/bfx_websocket_bucket.py b/bfxapi/websocket/client/bfx_websocket_bucket.py index 927a3e4..1f4dcde 100644 --- a/bfxapi/websocket/client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/client/bfx_websocket_bucket.py @@ -1,5 +1,3 @@ -from typing import Literal, TypeVar, Callable, cast - import asyncio, json, uuid, websockets from ..handlers import PublicChannelsHandler @@ -8,16 +6,14 @@ from ..exceptions import ConnectionNotOpen, TooManySubscriptions _HEARTBEAT = "hb" -F = TypeVar("F", bound=Callable[..., Literal[None]]) - -def _require_websocket_connection(function: F) -> F: +def _require_websocket_connection(function): async def wrapper(self, *args, **kwargs): if self.websocket is None or not self.websocket.open: raise ConnectionNotOpen("No open connection with the server.") await function(self, *args, **kwargs) - return cast(F, wrapper) + return wrapper class BfxWebSocketBucket: VERSION = 2 diff --git a/bfxapi/websocket/client/bfx_websocket_client.py b/bfxapi/websocket/client/bfx_websocket_client.py index c59f1b1..2a5a935 100644 --- a/bfxapi/websocket/client/bfx_websocket_client.py +++ b/bfxapi/websocket/client/bfx_websocket_client.py @@ -1,5 +1,3 @@ -from typing import cast - from collections import namedtuple from datetime import datetime @@ -8,7 +6,7 @@ import traceback, json, asyncio, hmac, hashlib, time, socket, random, websockets from pyee.asyncio import AsyncIOEventEmitter -from .bfx_websocket_bucket import _HEARTBEAT, F, _require_websocket_connection, BfxWebSocketBucket +from .bfx_websocket_bucket import _HEARTBEAT, _require_websocket_connection, BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs from ..handlers import PublicChannelsHandler, AuthEventsHandler @@ -19,7 +17,7 @@ from ...utils.json_encoder import JSONEncoder from ...utils.logger import ColorLogger, FileLogger -def _require_websocket_authentication(function: F) -> F: +def _require_websocket_authentication(function): async def wrapper(self, *args, **kwargs): if hasattr(self, "authentication") and not self.authentication: raise WebSocketAuthenticationRequired("To perform this action you need to " \ @@ -27,7 +25,7 @@ def _require_websocket_authentication(function: F) -> F: await _require_websocket_connection(function)(self, *args, **kwargs) - return cast(F, wrapper) + return wrapper class _Delay: BACKOFF_MIN, BACKOFF_MAX = 1.92, 60.0 From b12fedb7a3c74062d904c6e50d88ad71254fc19c Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sat, 17 Jun 2023 22:20:31 +0200 Subject: [PATCH 11/65] Replace use of asyncio.locks.Event with asyncio.locks.Condition in bfx_websocket_bucket.py. --- .../websocket/client/bfx_websocket_bucket.py | 62 ++++++++++--------- .../websocket/client/bfx_websocket_client.py | 34 +++++----- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/bfxapi/websocket/client/bfx_websocket_bucket.py b/bfxapi/websocket/client/bfx_websocket_bucket.py index 1f4dcde..3cca1a8 100644 --- a/bfxapi/websocket/client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/client/bfx_websocket_bucket.py @@ -23,48 +23,44 @@ class BfxWebSocketBucket: def __init__(self, host, event_emitter, events_per_subscription): self.host, self.event_emitter, self.events_per_subscription = host, event_emitter, events_per_subscription self.websocket, self.subscriptions, self.pendings = None, {}, [] - self.on_open_event = asyncio.locks.Event() + self.condition = asyncio.locks.Condition() self.handler = PublicChannelsHandler(event_emitter=self.event_emitter, \ events_per_subscription=self.events_per_subscription) async def connect(self): - async def _connection(): - async with websockets.connect(self.host) as websocket: - self.websocket = websocket - self.on_open_event.set() - await self.__recover_state() + async with websockets.connect(self.host) as websocket: + self.websocket = websocket - async for message in websocket: - message = json.loads(message) + await self.__recover_state() - if isinstance(message, dict): - if message["event"] == "subscribed" and (chan_id := message["chanId"]): - self.pendings = [ pending \ - for pending in self.pendings if pending["subId"] != message["subId"] ] + async with self.condition: + self.condition.notify() - self.subscriptions[chan_id] = message + async for message in websocket: + message = json.loads(message) - sub_id = message["subId"] + if isinstance(message, dict): + if message["event"] == "subscribed" and (chan_id := message["chanId"]): + self.pendings = [ pending \ + for pending in self.pendings if pending["subId"] != message["subId"] ] - if "subscribed" not in self.events_per_subscription.get(sub_id, []): - self.events_per_subscription.setdefault(sub_id, []).append("subscribed") - self.event_emitter.emit("subscribed", message) - elif message["event"] == "unsubscribed" and (chan_id := message["chanId"]): - if message["status"] == "OK": - del self.subscriptions[chan_id] - elif message["event"] == "error": - self.event_emitter.emit("wss-error", message["code"], message["msg"]) + self.subscriptions[chan_id] = message - if isinstance(message, list): - if (chan_id := message[0]) and message[1] != _HEARTBEAT: - self.handler.handle(self.subscriptions[chan_id], message[1:]) + sub_id = message["subId"] - try: - await _connection() - except websockets.exceptions.ConnectionClosedError as error: - if error.code in (1006, 1012): - self.on_open_event.clear() + if "subscribed" not in self.events_per_subscription.get(sub_id, []): + self.events_per_subscription.setdefault(sub_id, []).append("subscribed") + self.event_emitter.emit("subscribed", message) + elif message["event"] == "unsubscribed" and (chan_id := message["chanId"]): + if message["status"] == "OK": + del self.subscriptions[chan_id] + elif message["event"] == "error": + self.event_emitter.emit("wss-error", message["code"], message["msg"]) + + if isinstance(message, list): + if (chan_id := message[0]) and message[1] != _HEARTBEAT: + self.handler.handle(self.subscriptions[chan_id], message[1:]) async def __recover_state(self): for pending in self.pendings: @@ -107,3 +103,9 @@ class BfxWebSocketBucket: for subscription in self.subscriptions.values(): if subscription["subId"] == sub_id: return subscription["chanId"] + + async def wait(self): + async with self.condition: + await self.condition.wait_for( + lambda: self.websocket is not None and \ + self.websocket.open) diff --git a/bfxapi/websocket/client/bfx_websocket_client.py b/bfxapi/websocket/client/bfx_websocket_client.py index 2a5a935..77ff235 100644 --- a/bfxapi/websocket/client/bfx_websocket_client.py +++ b/bfxapi/websocket/client/bfx_websocket_client.py @@ -138,7 +138,7 @@ class BfxWebSocketClient: tasks = [ asyncio.create_task(coroutine) for coroutine in coroutines ] if len(self.buckets) == 0 or \ - (await asyncio.gather(*[bucket.on_open_event.wait() for bucket in self.buckets])): + (await asyncio.gather(*[bucket.wait() for bucket in self.buckets])): self.event_emitter.emit("open") if self.credentials: @@ -184,34 +184,34 @@ class BfxWebSocketClient: try: await _connection() except (websockets.exceptions.ConnectionClosedError, socket.gaierror) as error: - if isinstance(error, websockets.exceptions.ConnectionClosedError): - if error.code in (1006, 1012): - if error.code == 1006: - self.logger.error("Connection lost: no close frame received " \ - "or sent (1006). Trying to reconnect...") + for task in tasks: + task.cancel() - if error.code == 1012: - self.logger.info("WSS server is about to restart, clients need " \ - "to reconnect (server sent 20051). Reconnection attempt in progress...") + if isinstance(error, websockets.exceptions.ConnectionClosedError) and error.code in (1006, 1012): + if error.code == 1006: + self.logger.error("Connection lost: no close frame received " \ + "or sent (1006). Trying to reconnect...") - for task in tasks: - task.cancel() + if error.code == 1012: + self.logger.info("WSS server is about to restart, clients need " \ + "to reconnect (server sent 20051). Reconnection attempt in progress...") - reconnection = Reconnection(status=True, attempts=1, timestamp=datetime.now()) + reconnection = Reconnection(status=True, attempts=1, timestamp=datetime.now()) - if self.wss_timeout is not None: - timer = asyncio.get_event_loop().call_later(self.wss_timeout, _on_wss_timeout) + if self.wss_timeout is not None: + timer = asyncio.get_event_loop().call_later(self.wss_timeout, _on_wss_timeout) - delay = _Delay(backoff_factor=1.618) + delay = _Delay(backoff_factor=1.618) - self.authentication = False + self.authentication = False elif isinstance(error, socket.gaierror) and reconnection.status: self.logger.warning(f"Reconnection attempt was unsuccessful (no.{reconnection.attempts}). " \ f"Next reconnection attempt in {delay.peek():.2f} seconds. (at the moment " \ f"the client has been offline for {datetime.now() - reconnection.timestamp})") reconnection = reconnection._replace(attempts=reconnection.attempts + 1) - else: raise error + else: + raise error if not reconnection.status: self.event_emitter.emit("disconnection", From d9733e8d3853e47480f93f78e340fcf276f8cf6b Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sat, 17 Jun 2023 22:24:27 +0200 Subject: [PATCH 12/65] Change visibility of decorators require_websocket_connection and require_websocket_authentication (and hardcode HEARTBEAT). --- bfxapi/websocket/client/bfx_websocket_bucket.py | 12 +++++------- bfxapi/websocket/client/bfx_websocket_client.py | 17 +++++++++-------- bfxapi/websocket/exceptions.py | 4 ++-- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/bfxapi/websocket/client/bfx_websocket_bucket.py b/bfxapi/websocket/client/bfx_websocket_bucket.py index 3cca1a8..ef262b8 100644 --- a/bfxapi/websocket/client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/client/bfx_websocket_bucket.py @@ -4,9 +4,7 @@ from ..handlers import PublicChannelsHandler from ..exceptions import ConnectionNotOpen, TooManySubscriptions -_HEARTBEAT = "hb" - -def _require_websocket_connection(function): +def require_websocket_connection(function): async def wrapper(self, *args, **kwargs): if self.websocket is None or not self.websocket.open: raise ConnectionNotOpen("No open connection with the server.") @@ -59,7 +57,7 @@ class BfxWebSocketBucket: self.event_emitter.emit("wss-error", message["code"], message["msg"]) if isinstance(message, list): - if (chan_id := message[0]) and message[1] != _HEARTBEAT: + if (chan_id := message[0]) and message[1] != "hb": self.handler.handle(self.subscriptions[chan_id], message[1:]) async def __recover_state(self): @@ -71,7 +69,7 @@ class BfxWebSocketBucket: self.subscriptions.clear() - @_require_websocket_connection + @require_websocket_connection async def subscribe(self, channel, sub_id=None, **kwargs): if len(self.subscriptions) + len(self.pendings) == BfxWebSocketBucket.MAXIMUM_SUBSCRIPTIONS_AMOUNT: raise TooManySubscriptions("The client has reached the maximum number of subscriptions.") @@ -88,14 +86,14 @@ class BfxWebSocketBucket: await self.websocket.send(json.dumps(subscription)) - @_require_websocket_connection + @require_websocket_connection async def unsubscribe(self, chan_id): await self.websocket.send(json.dumps({ "event": "unsubscribe", "chanId": chan_id })) - @_require_websocket_connection + @require_websocket_connection async def close(self, code=1000, reason=str()): await self.websocket.close(code=code, reason=reason) diff --git a/bfxapi/websocket/client/bfx_websocket_client.py b/bfxapi/websocket/client/bfx_websocket_client.py index 77ff235..916a722 100644 --- a/bfxapi/websocket/client/bfx_websocket_client.py +++ b/bfxapi/websocket/client/bfx_websocket_client.py @@ -6,24 +6,25 @@ import traceback, json, asyncio, hmac, hashlib, time, socket, random, websockets from pyee.asyncio import AsyncIOEventEmitter -from .bfx_websocket_bucket import _HEARTBEAT, _require_websocket_connection, BfxWebSocketBucket +from .bfx_websocket_bucket import require_websocket_connection, BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs from ..handlers import PublicChannelsHandler, AuthEventsHandler -from ..exceptions import WebSocketAuthenticationRequired, InvalidAuthenticationCredentials, EventNotSupported, \ +from ..exceptions import ActionRequiresAuthentication, InvalidAuthenticationCredentials, EventNotSupported, \ ZeroConnectionsError, ReconnectionTimeoutError, OutdatedClientVersion from ...utils.json_encoder import JSONEncoder from ...utils.logger import ColorLogger, FileLogger -def _require_websocket_authentication(function): +def require_websocket_authentication(function): async def wrapper(self, *args, **kwargs): if hasattr(self, "authentication") and not self.authentication: - raise WebSocketAuthenticationRequired("To perform this action you need to " \ + raise ActionRequiresAuthentication("To perform this action you need to " \ "authenticate using your API_KEY and API_SECRET.") - await _require_websocket_connection(function)(self, *args, **kwargs) + await require_websocket_connection(function) \ + (self, *args, **kwargs) return wrapper @@ -170,7 +171,7 @@ class BfxWebSocketClient: self.event_emitter.emit("wss-error", message["code"], message["msg"]) if isinstance(message, list): - if message[0] == 0 and message[1] != _HEARTBEAT: + if message[0] == 0 and message[1] != "hb": self.handler.handle(message[1], message[2]) while True: @@ -256,11 +257,11 @@ class BfxWebSocketClient: if self.websocket is not None and self.websocket.open: await self.websocket.close(code=code, reason=reason) - @_require_websocket_authentication + @require_websocket_authentication async def notify(self, info, message_id=None, **kwargs): await self.websocket.send(json.dumps([ 0, "n", message_id, { "type": "ucm-test", "info": info, **kwargs } ])) - @_require_websocket_authentication + @require_websocket_authentication async def __handle_websocket_input(self, event, data): await self.websocket.send(json.dumps([ 0, event, None, data], cls=JSONEncoder)) diff --git a/bfxapi/websocket/exceptions.py b/bfxapi/websocket/exceptions.py index e47a1f0..6ed6f3f 100644 --- a/bfxapi/websocket/exceptions.py +++ b/bfxapi/websocket/exceptions.py @@ -7,7 +7,7 @@ __all__ = [ "TooManySubscriptions", "ZeroConnectionsError", "ReconnectionTimeoutError", - "WebSocketAuthenticationRequired", + "ActionRequiresAuthentication", "InvalidAuthenticationCredentials", "EventNotSupported", "OutdatedClientVersion" @@ -38,7 +38,7 @@ class ReconnectionTimeoutError(BfxWebSocketException): This error indicates that the connection has been offline for too long without being able to reconnect. """ -class WebSocketAuthenticationRequired(BfxWebSocketException): +class ActionRequiresAuthentication(BfxWebSocketException): """ This error indicates an attempt to access a protected resource without logging in first. """ From fc843895646455334db2fb275d849068b3424f43 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 18 Jun 2023 00:54:56 +0200 Subject: [PATCH 13/65] Remove support for BfxWebSocketClient's instance variable events_for_subscription. --- .../websocket/client/bfx_websocket_bucket.py | 19 ++- .../websocket/client/bfx_websocket_client.py | 4 +- .../handlers/public_channels_handler.py | 115 +++++++++--------- 3 files changed, 67 insertions(+), 71 deletions(-) diff --git a/bfxapi/websocket/client/bfx_websocket_bucket.py b/bfxapi/websocket/client/bfx_websocket_bucket.py index ef262b8..ca8625e 100644 --- a/bfxapi/websocket/client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/client/bfx_websocket_bucket.py @@ -18,13 +18,14 @@ class BfxWebSocketBucket: MAXIMUM_SUBSCRIPTIONS_AMOUNT = 25 - def __init__(self, host, event_emitter, events_per_subscription): - self.host, self.event_emitter, self.events_per_subscription = host, event_emitter, events_per_subscription - self.websocket, self.subscriptions, self.pendings = None, {}, [] - self.condition = asyncio.locks.Condition() + def __init__(self, host, event_emitter): + self.host, self.websocket, self.event_emitter = \ + host, None, event_emitter - self.handler = PublicChannelsHandler(event_emitter=self.event_emitter, \ - events_per_subscription=self.events_per_subscription) + self.condition, self.subscriptions, self.pendings = \ + asyncio.locks.Condition(), {}, [] + + self.handler = PublicChannelsHandler(event_emitter=self.event_emitter) async def connect(self): async with websockets.connect(self.host) as websocket: @@ -45,11 +46,7 @@ class BfxWebSocketBucket: self.subscriptions[chan_id] = message - sub_id = message["subId"] - - if "subscribed" not in self.events_per_subscription.get(sub_id, []): - self.events_per_subscription.setdefault(sub_id, []).append("subscribed") - self.event_emitter.emit("subscribed", message) + self.event_emitter.emit("subscribed", message) elif message["event"] == "unsubscribed" and (chan_id := message["chanId"]): if message["status"] == "OK": del self.subscriptions[chan_id] diff --git a/bfxapi/websocket/client/bfx_websocket_client.py b/bfxapi/websocket/client/bfx_websocket_client.py index 916a722..515e11b 100644 --- a/bfxapi/websocket/client/bfx_websocket_client.py +++ b/bfxapi/websocket/client/bfx_websocket_client.py @@ -71,8 +71,6 @@ class BfxWebSocketClient: self.host, self.credentials, self.wss_timeout = host, credentials, wss_timeout - self.events_per_subscription = {} - self.event_emitter = AsyncIOEventEmitter() self.handler = AuthEventsHandler(event_emitter=self.event_emitter) @@ -102,7 +100,7 @@ class BfxWebSocketClient: "block the client with <429 Too Many Requests>.") for _ in range(connections): - self.buckets += [BfxWebSocketBucket(self.host, self.event_emitter, self.events_per_subscription)] + self.buckets += [BfxWebSocketBucket(self.host, self.event_emitter)] await self.__connect() diff --git a/bfxapi/websocket/handlers/public_channels_handler.py b/bfxapi/websocket/handlers/public_channels_handler.py index 6a4a919..4b09bf0 100644 --- a/bfxapi/websocket/handlers/public_channels_handler.py +++ b/bfxapi/websocket/handlers/public_channels_handler.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING, \ - Union, Dict, List, Any, cast + Union, List, Any, cast from bfxapi.types import serializers @@ -28,11 +28,8 @@ class PublicChannelsHandler: "liquidation_feed_update" ] - def __init__(self, - event_emitter: "EventEmitter", - events_per_subscription: Dict[str, List[str]]) -> None: - self.__event_emitter, self.__events_per_subscription = \ - event_emitter, events_per_subscription + def __init__(self, event_emitter: "EventEmitter") -> None: + self.__event_emitter = event_emitter def handle(self, subscription: "Subscription", stream: List[Any]) -> None: def _strip(subscription: "Subscription", *args: str) -> "_NoHeaderSubscription": @@ -46,97 +43,101 @@ class PublicChannelsHandler: elif subscription["channel"] == "trades": self.__trades_channel_handler(cast("Trades", _subscription), stream) elif subscription["channel"] == "book": - self.__book_channel_handler(cast("Book", _subscription), stream) + _subscription = cast("Book", _subscription) + + if _subscription["prec"] != "R0": + self.__book_channel_handler(_subscription, stream) + else: + self.__raw_book_channel_handler(_subscription, stream) elif subscription["channel"] == "candles": self.__candles_channel_handler(cast("Candles", _subscription), stream) elif subscription["channel"] == "status": self.__status_channel_handler(cast("Status", _subscription), stream) - def __emit(self, event: str, subscription: "_NoHeaderSubscription", data: Any) -> None: - sub_id, should_emit_event = subscription["subId"], True - - if event in PublicChannelsHandler.ONCE_PER_SUBSCRIPTION_EVENTS: - if sub_id not in self.__events_per_subscription: - self.__events_per_subscription[sub_id] = [ event ] - elif event not in self.__events_per_subscription[sub_id]: - self.__events_per_subscription[sub_id] += [ event ] - else: should_emit_event = False - - if should_emit_event: - self.__event_emitter.emit(event, subscription, data) - - def __ticker_channel_handler(self, subscription: "Ticker", stream: List[Any]) -> None: + def __ticker_channel_handler(self, subscription: "Ticker", stream: List[Any]): if subscription["symbol"].startswith("t"): - return self.__emit("t_ticker_update", subscription, \ + return self.__event_emitter.emit("t_ticker_update", subscription, \ serializers.TradingPairTicker.parse(*stream[0])) if subscription["symbol"].startswith("f"): - return self.__emit("f_ticker_update", subscription, \ + return self.__event_emitter.emit("f_ticker_update", subscription, \ serializers.FundingCurrencyTicker.parse(*stream[0])) - def __trades_channel_handler(self, subscription: "Trades", stream: List[Any]) -> None: + def __trades_channel_handler(self, subscription: "Trades", stream: List[Any]): if (event := stream[0]) and event in [ "te", "tu", "fte", "ftu" ]: events = { "te": "t_trade_execution", "tu": "t_trade_execution_update", \ "fte": "f_trade_execution", "ftu": "f_trade_execution_update" } if subscription["symbol"].startswith("t"): - return self.__emit(events[event], subscription, \ + return self.__event_emitter.emit(events[event], subscription, \ serializers.TradingPairTrade.parse(*stream[1])) if subscription["symbol"].startswith("f"): - return self.__emit(events[event], subscription, \ + return self.__event_emitter.emit(events[event], subscription, \ serializers.FundingCurrencyTrade.parse(*stream[1])) if subscription["symbol"].startswith("t"): - return self.__emit("t_trades_snapshot", subscription, \ + return self.__event_emitter.emit("t_trades_snapshot", subscription, \ [ serializers.TradingPairTrade.parse(*sub_stream) \ for sub_stream in stream[0] ]) if subscription["symbol"].startswith("f"): - return self.__emit("f_trades_snapshot", subscription, \ + return self.__event_emitter.emit("f_trades_snapshot", subscription, \ [ serializers.FundingCurrencyTrade.parse(*sub_stream) \ for sub_stream in stream[0] ]) - def __book_channel_handler(self, subscription: "Book", stream: List[Any]) -> None: - symbol = subscription["symbol"] + def __book_channel_handler(self, subscription: "Book", stream: List[Any]): + if subscription["symbol"].startswith("t"): + if all(isinstance(sub_stream, list) for sub_stream in stream[0]): + return self.__event_emitter.emit("t_book_snapshot", subscription, \ + [ serializers.TradingPairBook.parse(*sub_stream) \ + for sub_stream in stream[0] ]) - is_raw_book = subscription["prec"] == "R0" + return self.__event_emitter.emit("t_book_update", subscription, \ + serializers.TradingPairBook.parse(*stream[0])) - serializer = { - "t": is_raw_book and serializers.TradingPairRawBook \ - or serializers.TradingPairBook, - "f": is_raw_book and serializers.FundingCurrencyRawBook \ - or serializers.FundingCurrencyBook - }[symbol[0]] + if subscription["symbol"].startswith("f"): + if all(isinstance(sub_stream, list) for sub_stream in stream[0]): + return self.__event_emitter.emit("f_book_snapshot", subscription, \ + [ serializers.FundingCurrencyBook.parse(*sub_stream) \ + for sub_stream in stream[0] ]) + return self.__event_emitter.emit("f_book_update", subscription, \ + serializers.FundingCurrencyBook.parse(*stream[0])) + + def __raw_book_channel_handler(self, subscription: "Book", stream: List[Any]): + if subscription["symbol"].startswith("t"): + if all(isinstance(sub_stream, list) for sub_stream in stream[0]): + self.__event_emitter.emit("t_raw_book_snapshot", subscription, \ + [ serializers.TradingPairRawBook.parse(*sub_stream) \ + for sub_stream in stream[0] ]) + + return self.__event_emitter.emit("t_raw_book_update", subscription, \ + serializers.TradingPairRawBook.parse(*stream[0])) + + if subscription["symbol"].startswith("f"): + if all(isinstance(sub_stream, list) for sub_stream in stream[0]): + return self.__event_emitter.emit("f_raw_book_snapshot", subscription, \ + [ serializers.FundingCurrencyRawBook.parse(*sub_stream) \ + for sub_stream in stream[0] ]) + + return self.__event_emitter.emit("f_raw_book_update", subscription, \ + serializers.FundingCurrencyRawBook.parse(*stream[0])) + + def __candles_channel_handler(self, subscription: "Candles", stream: List[Any]): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): - event = symbol[0] + "_" + \ - (is_raw_book and "raw_book" or "book") + "_snapshot" - - return self.__emit(event, subscription, \ - [ serializer.parse(*sub_stream) \ - for sub_stream in stream[0] ]) - - event = symbol[0] + "_" + \ - (is_raw_book and "raw_book" or "book") + "_update" - - return self.__emit(event, subscription, \ - serializer.parse(*stream[0])) - - def __candles_channel_handler(self, subscription: "Candles", stream: List[Any]) -> None: - if all(isinstance(sub_stream, list) for sub_stream in stream[0]): - return self.__emit("candles_snapshot", subscription, \ + return self.__event_emitter.emit("candles_snapshot", subscription, \ [ serializers.Candle.parse(*sub_stream) \ for sub_stream in stream[0] ]) - return self.__emit("candles_update", subscription, \ + return self.__event_emitter.emit("candles_update", subscription, \ serializers.Candle.parse(*stream[0])) - def __status_channel_handler(self, subscription: "Status", stream: List[Any]) -> None: + def __status_channel_handler(self, subscription: "Status", stream: List[Any]): if subscription["key"].startswith("deriv:"): - return self.__emit("derivatives_status_update", subscription, \ + return self.__event_emitter.emit("derivatives_status_update", subscription, \ serializers.DerivativesStatus.parse(*stream[0])) if subscription["key"].startswith("liq:"): - return self.__emit("liquidation_feed_update", subscription, \ + return self.__event_emitter.emit("liquidation_feed_update", subscription, \ serializers.Liquidation.parse(*stream[0][0])) From 1d911a250cae6194ac51539cac757161cd321abe Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 18 Jun 2023 01:02:59 +0200 Subject: [PATCH 14/65] Rename bfxapi.websocket.client to _client and bfxapi.websocket.handlers to _handlers (according to PEP8). --- bfxapi/websocket/__init__.py | 2 +- bfxapi/websocket/{client => _client}/__init__.py | 0 bfxapi/websocket/{client => _client}/bfx_websocket_bucket.py | 2 +- bfxapi/websocket/{client => _client}/bfx_websocket_client.py | 2 +- bfxapi/websocket/{client => _client}/bfx_websocket_inputs.py | 0 bfxapi/websocket/{handlers => _handlers}/__init__.py | 0 bfxapi/websocket/{handlers => _handlers}/auth_events_handler.py | 0 .../{handlers => _handlers}/public_channels_handler.py | 0 setup.py | 2 +- 9 files changed, 4 insertions(+), 4 deletions(-) rename bfxapi/websocket/{client => _client}/__init__.py (100%) rename bfxapi/websocket/{client => _client}/bfx_websocket_bucket.py (98%) rename bfxapi/websocket/{client => _client}/bfx_websocket_client.py (99%) rename bfxapi/websocket/{client => _client}/bfx_websocket_inputs.py (100%) rename bfxapi/websocket/{handlers => _handlers}/__init__.py (100%) rename bfxapi/websocket/{handlers => _handlers}/auth_events_handler.py (100%) rename bfxapi/websocket/{handlers => _handlers}/public_channels_handler.py (100%) diff --git a/bfxapi/websocket/__init__.py b/bfxapi/websocket/__init__.py index 52e603a..f1ed659 100644 --- a/bfxapi/websocket/__init__.py +++ b/bfxapi/websocket/__init__.py @@ -1 +1 @@ -from .client import BfxWebSocketClient, BfxWebSocketBucket, BfxWebSocketInputs +from ._client import BfxWebSocketClient, BfxWebSocketBucket, BfxWebSocketInputs diff --git a/bfxapi/websocket/client/__init__.py b/bfxapi/websocket/_client/__init__.py similarity index 100% rename from bfxapi/websocket/client/__init__.py rename to bfxapi/websocket/_client/__init__.py diff --git a/bfxapi/websocket/client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py similarity index 98% rename from bfxapi/websocket/client/bfx_websocket_bucket.py rename to bfxapi/websocket/_client/bfx_websocket_bucket.py index ca8625e..2cc421b 100644 --- a/bfxapi/websocket/client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -1,6 +1,6 @@ import asyncio, json, uuid, websockets -from ..handlers import PublicChannelsHandler +from .._handlers import PublicChannelsHandler from ..exceptions import ConnectionNotOpen, TooManySubscriptions diff --git a/bfxapi/websocket/client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py similarity index 99% rename from bfxapi/websocket/client/bfx_websocket_client.py rename to bfxapi/websocket/_client/bfx_websocket_client.py index 515e11b..8abb3c2 100644 --- a/bfxapi/websocket/client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -9,7 +9,7 @@ from pyee.asyncio import AsyncIOEventEmitter from .bfx_websocket_bucket import require_websocket_connection, BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs -from ..handlers import PublicChannelsHandler, AuthEventsHandler +from .._handlers import PublicChannelsHandler, AuthEventsHandler from ..exceptions import ActionRequiresAuthentication, InvalidAuthenticationCredentials, EventNotSupported, \ ZeroConnectionsError, ReconnectionTimeoutError, OutdatedClientVersion diff --git a/bfxapi/websocket/client/bfx_websocket_inputs.py b/bfxapi/websocket/_client/bfx_websocket_inputs.py similarity index 100% rename from bfxapi/websocket/client/bfx_websocket_inputs.py rename to bfxapi/websocket/_client/bfx_websocket_inputs.py diff --git a/bfxapi/websocket/handlers/__init__.py b/bfxapi/websocket/_handlers/__init__.py similarity index 100% rename from bfxapi/websocket/handlers/__init__.py rename to bfxapi/websocket/_handlers/__init__.py diff --git a/bfxapi/websocket/handlers/auth_events_handler.py b/bfxapi/websocket/_handlers/auth_events_handler.py similarity index 100% rename from bfxapi/websocket/handlers/auth_events_handler.py rename to bfxapi/websocket/_handlers/auth_events_handler.py diff --git a/bfxapi/websocket/handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py similarity index 100% rename from bfxapi/websocket/handlers/public_channels_handler.py rename to bfxapi/websocket/_handlers/public_channels_handler.py diff --git a/setup.py b/setup.py index 486db47..e884b88 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( }, packages=[ "bfxapi", "bfxapi.utils", "bfxapi.types", - "bfxapi.websocket", "bfxapi.websocket.client", "bfxapi.websocket.handlers", + "bfxapi.websocket", "bfxapi.websocket._client", "bfxapi.websocket._handlers", "bfxapi.rest", "bfxapi.rest.endpoints", "bfxapi.rest.middleware", ], install_requires=[ From 080ec40395ccf79536ffc6cabe1483bb4c2b8430 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 18 Jun 2023 03:56:07 +0200 Subject: [PATCH 15/65] Add sub-package bfxapi.websocket._event_emitter (with bfx_event_emitter.py). --- .../websocket/_client/bfx_websocket_client.py | 16 ++++---- bfxapi/websocket/_event_emitter/__init__.py | 1 + .../_event_emitter/bfx_event_emitter.py | 37 +++++++++++++++++++ .../_handlers/auth_events_handler.py | 16 ++++---- .../_handlers/public_channels_handler.py | 4 +- setup.py | 3 +- 6 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 bfxapi/websocket/_event_emitter/__init__.py create mode 100644 bfxapi/websocket/_event_emitter/bfx_event_emitter.py diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 8abb3c2..28fc9ea 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -4,8 +4,6 @@ from datetime import datetime import traceback, json, asyncio, hmac, hashlib, time, socket, random, websockets -from pyee.asyncio import AsyncIOEventEmitter - from .bfx_websocket_bucket import require_websocket_connection, BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs @@ -13,6 +11,8 @@ from .._handlers import PublicChannelsHandler, AuthEventsHandler from ..exceptions import ActionRequiresAuthentication, InvalidAuthenticationCredentials, EventNotSupported, \ ZeroConnectionsError, ReconnectionTimeoutError, OutdatedClientVersion +from .._event_emitter import BfxEventEmitter + from ...utils.json_encoder import JSONEncoder from ...utils.logger import ColorLogger, FileLogger @@ -54,14 +54,14 @@ class BfxWebSocketClient: MAXIMUM_CONNECTIONS_AMOUNT = 20 - ONCE_EVENTS = [ + __ONCE_EVENTS = [ "open", "authenticated", "disconnection", *AuthEventsHandler.ONCE_EVENTS ] EVENTS = [ "subscribed", "wss-error", - *ONCE_EVENTS, + *__ONCE_EVENTS, *PublicChannelsHandler.EVENTS, *AuthEventsHandler.ON_EVENTS ] @@ -71,7 +71,9 @@ class BfxWebSocketClient: self.host, self.credentials, self.wss_timeout = host, credentials, wss_timeout - self.event_emitter = AsyncIOEventEmitter() + self.event_emitter = BfxEventEmitter(targets= \ + PublicChannelsHandler.ONCE_PER_SUBSCRIPTION + \ + ["subscribed"]) self.handler = AuthEventsHandler(event_emitter=self.event_emitter) @@ -267,10 +269,10 @@ class BfxWebSocketClient: for event in events: if event not in BfxWebSocketClient.EVENTS: raise EventNotSupported(f"Event <{event}> is not supported. To get a list " \ - "of available events print BfxWebSocketClient.EVENTS") + "of available events see BfxWebSocketClient.EVENTS.") def _register_event(event, function): - if event in BfxWebSocketClient.ONCE_EVENTS: + if event in BfxWebSocketClient.__ONCE_EVENTS: self.event_emitter.once(event, function) else: self.event_emitter.on(event, function) diff --git a/bfxapi/websocket/_event_emitter/__init__.py b/bfxapi/websocket/_event_emitter/__init__.py new file mode 100644 index 0000000..66f58ae --- /dev/null +++ b/bfxapi/websocket/_event_emitter/__init__.py @@ -0,0 +1 @@ +from .bfx_event_emitter import BfxEventEmitter diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py new file mode 100644 index 0000000..9ebabd2 --- /dev/null +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -0,0 +1,37 @@ +from typing import \ + TYPE_CHECKING, List, Dict, Any + +from collections import defaultdict + +from pyee.asyncio import AsyncIOEventEmitter + +if TYPE_CHECKING: + from bfxapi.websocket.subscriptions import Subscription + +class BfxEventEmitter(AsyncIOEventEmitter): + def __init__(self, targets: List[str]) -> None: + super().__init__() + + self.__targets = targets + + self.__log: Dict[str, List[str]] = \ + defaultdict(lambda: [ ]) + + def emit(self, + event: str, + *args: Any, + **kwargs: Any) -> bool: + if event in self.__targets: + subscription: "Subscription" = args[0] + + sub_id = subscription["subId"] + + if event in self.__log[sub_id]: + with self._lock: + listeners = self._events.get(event) + + return bool(listeners) + + self.__log[sub_id] += [ event ] + + return super().emit(event, *args, **kwargs) diff --git a/bfxapi/websocket/_handlers/auth_events_handler.py b/bfxapi/websocket/_handlers/auth_events_handler.py index f411e7b..7028043 100644 --- a/bfxapi/websocket/_handlers/auth_events_handler.py +++ b/bfxapi/websocket/_handlers/auth_events_handler.py @@ -12,12 +12,12 @@ if TYPE_CHECKING: from pyee.base import EventEmitter class AuthEventsHandler: - __once_abbreviations = { + __ONCE_ABBREVIATIONS = { "os": "order_snapshot", "ps": "position_snapshot", "fos": "funding_offer_snapshot", "fcs": "funding_credit_snapshot", "fls": "funding_loan_snapshot", "ws": "wallet_snapshot" } - __on_abbreviations = { + __ON_ABBREVIATIONS = { "on": "order_new", "ou": "order_update", "oc": "order_cancel", "pn": "position_new", "pu": "position_update", "pc": "position_close", "fon": "funding_offer_new", "fou": "funding_offer_update", "foc": "funding_offer_cancel", @@ -26,17 +26,17 @@ class AuthEventsHandler: "te": "trade_execution", "tu": "trade_execution_update", "wu": "wallet_update" } - __abbreviations = { - **__once_abbreviations, - **__on_abbreviations + __ABBREVIATIONS = { + **__ONCE_ABBREVIATIONS, + **__ON_ABBREVIATIONS } ONCE_EVENTS = [ - *list(__once_abbreviations.values()) + *list(__ONCE_ABBREVIATIONS.values()) ] ON_EVENTS = [ - *list(__on_abbreviations.values()), + *list(__ON_ABBREVIATIONS.values()), "notification", "on-req-notification", "ou-req-notification", "oc-req-notification", "fon-req-notification", "foc-req-notification" ] @@ -60,7 +60,7 @@ class AuthEventsHandler: for abbrevations, serializer in self.__serializers.items(): if abbrevation in abbrevations: - event = AuthEventsHandler.__abbreviations[abbrevation] + event = AuthEventsHandler.__ABBREVIATIONS[abbrevation] if all(isinstance(sub_stream, list) for sub_stream in stream): data = [ serializer.parse(*sub_stream) for sub_stream in stream ] diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index 4b09bf0..87c46e4 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -13,14 +13,14 @@ if TYPE_CHECKING: Union[Ticker, Trades, Book, Candles, Status] class PublicChannelsHandler: - ONCE_PER_SUBSCRIPTION_EVENTS = [ + ONCE_PER_SUBSCRIPTION = [ "t_trades_snapshot", "f_trades_snapshot", "t_book_snapshot", "f_book_snapshot", "t_raw_book_snapshot", "f_raw_book_snapshot", "candles_snapshot" ] EVENTS = [ - *ONCE_PER_SUBSCRIPTION_EVENTS, + *ONCE_PER_SUBSCRIPTION, "t_ticker_update", "f_ticker_update", "t_trade_execution", "t_trade_execution_update", "f_trade_execution", "f_trade_execution_update", "t_book_update", "f_book_update", "t_raw_book_update", diff --git a/setup.py b/setup.py index e884b88..8538ba9 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,8 @@ setup( }, packages=[ "bfxapi", "bfxapi.utils", "bfxapi.types", - "bfxapi.websocket", "bfxapi.websocket._client", "bfxapi.websocket._handlers", + "bfxapi.websocket", "bfxapi.websocket._client", "bfxapi.websocket._handlers", + "bfxapi.websocket._event_emitter", "bfxapi.rest", "bfxapi.rest.endpoints", "bfxapi.rest.middleware", ], install_requires=[ From 8b196b8f9ca5011f63832b5bcd40c9d30225fc79 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 18 Jun 2023 17:44:09 +0200 Subject: [PATCH 16/65] Add type hinting support to bfxapi.websocket.client.bfx_websocket_bucket. --- .../websocket/_client/bfx_websocket_bucket.py | 147 ++++++++++-------- bfxapi/websocket/_connection.py | 39 +++++ 2 files changed, 121 insertions(+), 65 deletions(-) create mode 100644 bfxapi/websocket/_connection.py diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 2cc421b..e8eddc7 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -1,106 +1,123 @@ +from typing import \ + TYPE_CHECKING, Optional, Dict, List, Any, cast + import asyncio, json, uuid, websockets -from .._handlers import PublicChannelsHandler +from bfxapi.websocket._connection import Connection +from bfxapi.websocket._handlers import PublicChannelsHandler +from bfxapi.websocket.exceptions import TooManySubscriptions -from ..exceptions import ConnectionNotOpen, TooManySubscriptions +if TYPE_CHECKING: + from bfxapi.websocket.subscriptions import Subscription + from websockets.client import WebSocketClientProtocol + from pyee import EventEmitter -def require_websocket_connection(function): - async def wrapper(self, *args, **kwargs): - if self.websocket is None or not self.websocket.open: - raise ConnectionNotOpen("No open connection with the server.") - - await function(self, *args, **kwargs) - - return wrapper - -class BfxWebSocketBucket: +class BfxWebSocketBucket(Connection): VERSION = 2 MAXIMUM_SUBSCRIPTIONS_AMOUNT = 25 - def __init__(self, host, event_emitter): - self.host, self.websocket, self.event_emitter = \ - host, None, event_emitter + def __init__(self, host: str, event_emitter: "EventEmitter") -> None: + super().__init__(host) - self.condition, self.subscriptions, self.pendings = \ - asyncio.locks.Condition(), {}, [] + self.__event_emitter = event_emitter + self.__pendings: List[Dict[str, Any]] = [ ] + self.__subscriptions: Dict[int, "Subscription"] = { } - self.handler = PublicChannelsHandler(event_emitter=self.event_emitter) + self.__condition = asyncio.locks.Condition() - async def connect(self): - async with websockets.connect(self.host) as websocket: - self.websocket = websocket + self.__handler = PublicChannelsHandler( \ + event_emitter=self.__event_emitter) + + @property + def pendings(self) -> List[Dict[str, Any]]: + return self.__pendings + + @property + def subscriptions(self) -> Dict[int, "Subscription"]: + return self.__subscriptions + + async def connect(self) -> None: + async with websockets.client.connect(self._host) as websocket: + self._websocket = websocket await self.__recover_state() - async with self.condition: - self.condition.notify() + async with self.__condition: + self.__condition.notify(1) - async for message in websocket: + async for message in self._websocket: message = json.loads(message) if isinstance(message, dict): if message["event"] == "subscribed" and (chan_id := message["chanId"]): - self.pendings = [ pending \ - for pending in self.pendings if pending["subId"] != message["subId"] ] + self.__pendings = [ pending \ + for pending in self.__pendings \ + if pending["subId"] != message["subId"] ] - self.subscriptions[chan_id] = message + self.__subscriptions[chan_id] = cast("Subscription", message) - self.event_emitter.emit("subscribed", message) + self.__event_emitter.emit("subscribed", message) elif message["event"] == "unsubscribed" and (chan_id := message["chanId"]): if message["status"] == "OK": - del self.subscriptions[chan_id] + del self.__subscriptions[chan_id] elif message["event"] == "error": - self.event_emitter.emit("wss-error", message["code"], message["msg"]) + self.__event_emitter.emit( \ + "wss-error", message["code"], message["msg"]) if isinstance(message, list): - if (chan_id := message[0]) and message[1] != "hb": - self.handler.handle(self.subscriptions[chan_id], message[1:]) + if (chan_id := message[0]) and message[1] != Connection.HEARTBEAT: + self.__handler.handle(self.__subscriptions[chan_id], message[1:]) - async def __recover_state(self): - for pending in self.pendings: - await self.websocket.send(json.dumps(pending)) + async def __recover_state(self) -> None: + for pending in self.__pendings: + await self._websocket.send( \ + json.dumps(pending)) - for _, subscription in self.subscriptions.items(): - await self.subscribe(sub_id=subscription.pop("subId"), **subscription) + for _, subscription in self.__subscriptions.items(): + _subscription = cast(Dict[str, Any], subscription) - self.subscriptions.clear() + await self.subscribe( \ + sub_id=_subscription.pop("subId"), **_subscription) - @require_websocket_connection - async def subscribe(self, channel, sub_id=None, **kwargs): - if len(self.subscriptions) + len(self.pendings) == BfxWebSocketBucket.MAXIMUM_SUBSCRIPTIONS_AMOUNT: + self.__subscriptions.clear() + + @Connection.require_websocket_connection + async def subscribe(self, + channel: str, + sub_id: Optional[str] = None, + **kwargs: Any) -> None: + if len(self.__subscriptions) + len(self.__pendings) \ + == BfxWebSocketBucket.MAXIMUM_SUBSCRIPTIONS_AMOUNT: raise TooManySubscriptions("The client has reached the maximum number of subscriptions.") - subscription = { - **kwargs, + subscription = \ + { **kwargs, "event": "subscribe", "channel": channel } - "event": "subscribe", - "channel": channel, - "subId": sub_id or str(uuid.uuid4()), - } + subscription["subId"] = sub_id or str(uuid.uuid4()) - self.pendings.append(subscription) + self.__pendings.append(subscription) - await self.websocket.send(json.dumps(subscription)) + await self._websocket.send( \ + json.dumps(subscription)) - @require_websocket_connection - async def unsubscribe(self, chan_id): - await self.websocket.send(json.dumps({ - "event": "unsubscribe", - "chanId": chan_id - })) + @Connection.require_websocket_connection + async def unsubscribe(self, chan_id: int) -> None: + await self._websocket.send(json.dumps( \ + { "event": "unsubscribe", "chanId": chan_id })) - @require_websocket_connection - async def close(self, code=1000, reason=str()): - await self.websocket.close(code=code, reason=reason) + @Connection.require_websocket_connection + async def close(self, code: int = 1000, reason: str = str()) -> None: + await self._websocket.close(code=code, reason=reason) - def get_chan_id(self, sub_id): - for subscription in self.subscriptions.values(): + def get_chan_id(self, sub_id: str) -> Optional[int]: + for subscription in self.__subscriptions.values(): if subscription["subId"] == sub_id: return subscription["chanId"] - async def wait(self): - async with self.condition: - await self.condition.wait_for( - lambda: self.websocket is not None and \ - self.websocket.open) + return None + + async def wait(self) -> None: + async with self.__condition: + await self.__condition.wait_for( + lambda: self.open) diff --git a/bfxapi/websocket/_connection.py b/bfxapi/websocket/_connection.py new file mode 100644 index 0000000..7908ca7 --- /dev/null +++ b/bfxapi/websocket/_connection.py @@ -0,0 +1,39 @@ +from typing import \ + TYPE_CHECKING, Optional, cast + +from bfxapi.websocket.exceptions import \ + ConnectionNotOpen + +if TYPE_CHECKING: + from websockets.client import WebSocketClientProtocol + +class Connection: + HEARTBEAT = "hb" + + def __init__(self, host: str) -> None: + self._host = host + + self.__protocol: Optional["WebSocketClientProtocol"] = None + + @property + def open(self) -> bool: + return self.__protocol is not None and \ + self.__protocol.open + + @property + def _websocket(self) -> "WebSocketClientProtocol": + return cast("WebSocketClientProtocol", self.__protocol) + + @_websocket.setter + def _websocket(self, protocol: "WebSocketClientProtocol") -> None: + self.__protocol = protocol + + @staticmethod + def require_websocket_connection(function): + async def wrapper(self, *args, **kwargs): + if self.open: + return await function(self, *args, **kwargs) + + raise ConnectionNotOpen("No open connection with the server.") + + return wrapper From f1e678e0438d25137a941323f684a7ee5ac9bf67 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 19 Jun 2023 04:57:33 +0200 Subject: [PATCH 17/65] Add type hinting support to bfxapi.websocket.client.bfx_websocket_client. --- bfxapi/client.py | 25 +- bfxapi/rest/endpoints/bfx_rest_interface.py | 4 +- .../websocket/_client/bfx_websocket_bucket.py | 24 +- .../websocket/_client/bfx_websocket_client.py | 441 ++++++++++-------- bfxapi/websocket/_connection.py | 99 ++-- 5 files changed, 341 insertions(+), 252 deletions(-) diff --git a/bfxapi/client.py b/bfxapi/client.py index dac3649..5e5292d 100644 --- a/bfxapi/client.py +++ b/bfxapi/client.py @@ -9,29 +9,16 @@ class Client: self, api_key: Optional[str] = None, api_secret: Optional[str] = None, - filters: Optional[List[str]] = None, *, rest_host: str = REST_HOST, wss_host: str = WSS_HOST, + filters: Optional[List[str]] = None, wss_timeout: Optional[float] = 60 * 15, log_filename: Optional[str] = None, log_level: Literal["ERROR", "WARNING", "INFO", "DEBUG"] = "INFO" - ): - credentials = None + ) -> None: + self.rest = BfxRestInterface(rest_host, api_key, api_secret) - if api_key and api_secret: - credentials = { "api_key": api_key, "api_secret": api_secret, "filters": filters } - - self.rest = BfxRestInterface( - host=rest_host, - credentials=credentials - ) - - self.wss = BfxWebSocketClient( - host=wss_host, - credentials=credentials, - wss_timeout=wss_timeout, - log_filename=log_filename, - log_level=log_level - ) - \ No newline at end of file + self.wss = BfxWebSocketClient(wss_host, api_key, api_secret, + filters=filters, wss_timeout=wss_timeout, log_filename=log_filename, + log_level=log_level) diff --git a/bfxapi/rest/endpoints/bfx_rest_interface.py b/bfxapi/rest/endpoints/bfx_rest_interface.py index 73ec603..1d17d26 100644 --- a/bfxapi/rest/endpoints/bfx_rest_interface.py +++ b/bfxapi/rest/endpoints/bfx_rest_interface.py @@ -5,9 +5,7 @@ from .rest_merchant_endpoints import RestMerchantEndpoints class BfxRestInterface: VERSION = 2 - def __init__(self, host, credentials = None): - api_key, api_secret = (credentials['api_key'], credentials['api_secret']) if credentials else (None, None) - + def __init__(self, host, api_key = None, api_secret = None): self.public = RestPublicEndpoints(host=host) self.auth = RestAuthEndpoints(host=host, api_key=api_key, api_secret=api_secret) self.merchant = RestMerchantEndpoints(host=host, api_key=api_key, api_secret=api_secret) diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index e8eddc7..c8dcf97 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -1,7 +1,9 @@ from typing import \ TYPE_CHECKING, Optional, Dict, List, Any, cast -import asyncio, json, uuid, websockets +import asyncio, json, uuid + +from websockets.legacy.client import connect as _websockets__connect from bfxapi.websocket._connection import Connection from bfxapi.websocket._handlers import PublicChannelsHandler @@ -38,7 +40,7 @@ class BfxWebSocketBucket(Connection): return self.__subscriptions async def connect(self) -> None: - async with websockets.client.connect(self._host) as websocket: + async with _websockets__connect(self._host) as websocket: self._websocket = websocket await self.__recover_state() @@ -102,20 +104,26 @@ class BfxWebSocketBucket(Connection): json.dumps(subscription)) @Connection.require_websocket_connection - async def unsubscribe(self, chan_id: int) -> None: - await self._websocket.send(json.dumps( \ - { "event": "unsubscribe", "chanId": chan_id })) + async def unsubscribe(self, sub_id: str) -> None: + for subscription in self.__subscriptions.values(): + if subscription["subId"] == sub_id: + data = { "event": "unsubscribe", \ + "chanId": subscription["subId"] } + + message = json.dumps(data) + + await self._websocket.send(message) @Connection.require_websocket_connection async def close(self, code: int = 1000, reason: str = str()) -> None: await self._websocket.close(code=code, reason=reason) - def get_chan_id(self, sub_id: str) -> Optional[int]: + def has(self, sub_id: str) -> bool: for subscription in self.__subscriptions.values(): if subscription["subId"] == sub_id: - return subscription["chanId"] + return True - return None + return False async def wait(self) -> None: async with self.__condition: diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 28fc9ea..38a2c37 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -1,55 +1,54 @@ -from collections import namedtuple +from typing import \ + TYPE_CHECKING, TypeVar, TypedDict,\ + Callable, Optional, Literal,\ + Tuple, List, Dict, \ + Any from datetime import datetime -import traceback, json, asyncio, hmac, hashlib, time, socket, random, websockets +from socket import gaierror -from .bfx_websocket_bucket import require_websocket_connection, BfxWebSocketBucket +import \ + traceback, json, asyncio, \ + hmac, hashlib, random, \ + websockets + +from websockets.legacy.client import connect as _websockets__connect + +from bfxapi.utils.json_encoder import JSONEncoder + +from bfxapi.utils.logger import \ + ColorLogger, FileLogger + +from bfxapi.websocket._handlers import \ + PublicChannelsHandler, AuthEventsHandler + +from bfxapi.websocket._connection import Connection + +from bfxapi.websocket._event_emitter import BfxEventEmitter + +from bfxapi.websocket.exceptions import \ + InvalidAuthenticationCredentials, EventNotSupported, ZeroConnectionsError, \ + ReconnectionTimeoutError, OutdatedClientVersion + +from .bfx_websocket_bucket import BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs -from .._handlers import PublicChannelsHandler, AuthEventsHandler -from ..exceptions import ActionRequiresAuthentication, InvalidAuthenticationCredentials, EventNotSupported, \ - ZeroConnectionsError, ReconnectionTimeoutError, OutdatedClientVersion -from .._event_emitter import BfxEventEmitter +if TYPE_CHECKING: + from logging import Logger -from ...utils.json_encoder import JSONEncoder + from asyncio import Task -from ...utils.logger import ColorLogger, FileLogger + _T = TypeVar("_T", bound=Callable[..., None]) -def require_websocket_authentication(function): - async def wrapper(self, *args, **kwargs): - if hasattr(self, "authentication") and not self.authentication: - raise ActionRequiresAuthentication("To perform this action you need to " \ - "authenticate using your API_KEY and API_SECRET.") + _Credentials = TypedDict("_Credentials", \ + { "api_key": str, "api_secret": str, "filters": Optional[List[str]] }) - await require_websocket_connection(function) \ - (self, *args, **kwargs) + _Reconnection = TypedDict("_Reconnection", + { "attempts": int, "reason": str, "timestamp": datetime }) - return wrapper - -class _Delay: - BACKOFF_MIN, BACKOFF_MAX = 1.92, 60.0 - - BACKOFF_INITIAL = 5.0 - - def __init__(self, backoff_factor): - self.__backoff_factor = backoff_factor - self.__backoff_delay = _Delay.BACKOFF_MIN - self.__initial_delay = random.random() * _Delay.BACKOFF_INITIAL - - def next(self): - backoff_delay = self.peek() - __backoff_delay = self.__backoff_delay * self.__backoff_factor - self.__backoff_delay = min(__backoff_delay, _Delay.BACKOFF_MAX) - - return backoff_delay - - def peek(self): - return (self.__backoff_delay == _Delay.BACKOFF_MIN) \ - and self.__initial_delay or self.__backoff_delay - -class BfxWebSocketClient: +class BfxWebSocketClient(Connection, Connection.Authenticable): VERSION = BfxWebSocketBucket.VERSION MAXIMUM_CONNECTIONS_AMOUNT = 20 @@ -66,223 +65,299 @@ class BfxWebSocketClient: *AuthEventsHandler.ON_EVENTS ] - def __init__(self, host, credentials, *, wss_timeout = 60 * 15, log_filename = None, log_level = "INFO"): - self.websocket, self.authentication, self.buckets = None, False, [] + def __init__(self, + host: str, + api_key: Optional[str] = None, + api_secret: Optional[str] = None, + *, + filters: Optional[List[str]] = None, + wss_timeout: Optional[float] = 60 * 15, + log_filename: Optional[str] = None, + log_level: Literal["ERROR", "WARNING", "INFO", "DEBUG"] = "INFO") -> None: + super().__init__(host) - self.host, self.credentials, self.wss_timeout = host, credentials, wss_timeout + self.__credentials: Optional["_Credentials"] = None - self.event_emitter = BfxEventEmitter(targets= \ + if api_key and api_secret: + self.__credentials = \ + { "api_key": api_key, "api_secret": api_secret, "filters": filters } + + self.__wss_timeout = wss_timeout + + self.__event_emitter = BfxEventEmitter(targets = \ PublicChannelsHandler.ONCE_PER_SUBSCRIPTION + \ ["subscribed"]) - self.handler = AuthEventsHandler(event_emitter=self.event_emitter) + self.__handler = AuthEventsHandler(\ + event_emitter=self.__event_emitter) - self.inputs = BfxWebSocketInputs(handle_websocket_input=self.__handle_websocket_input) + self.__inputs = BfxWebSocketInputs(\ + handle_websocket_input=self.__handle_websocket_input) + + self.__buckets: Dict[BfxWebSocketBucket, Optional["Task"]] = { } + + self.__reconnection: Optional[_Reconnection] = None + + self.__logger: "Logger" if log_filename is None: - self.logger = ColorLogger("BfxWebSocketClient", level=log_level) - else: self.logger = FileLogger("BfxWebSocketClient", level=log_level, filename=log_filename) + self.__logger = ColorLogger("BfxWebSocketClient", level=log_level) + else: self.__logger = FileLogger("BfxWebSocketClient", level=log_level, filename=log_filename) - self.event_emitter.add_listener("error", - lambda exception: self.logger.error(f"{type(exception).__name__}: {str(exception)}" + "\n" + + self.__event_emitter.add_listener("error", + lambda exception: self.__logger.error(f"{type(exception).__name__}: {str(exception)}" + "\n" + str().join(traceback.format_exception(type(exception), exception, exception.__traceback__))[:-1]) ) - def run(self, connections = 5): + @property + def inputs(self) -> BfxWebSocketInputs: + return self.__inputs + + def run(self, connections: int = 5) -> None: return asyncio.run(self.start(connections)) - async def start(self, connections = 5): + async def start(self, connections: int = 5) -> None: if connections == 0: - self.logger.info("With connections set to 0 it will not be possible to subscribe to any public channel. " \ - "Attempting a subscription will cause a ZeroConnectionsError to be thrown.") + self.__logger.info("With connections set to 0 it will not be possible to subscribe to any " \ + "public channel. Attempting a subscription will cause a ZeroConnectionsError to be thrown.") if connections > BfxWebSocketClient.MAXIMUM_CONNECTIONS_AMOUNT: - self.logger.warning(f"It is not safe to use more than {BfxWebSocketClient.MAXIMUM_CONNECTIONS_AMOUNT} " \ - f"buckets from the same connection ({connections} in use), the server could momentarily " \ - "block the client with <429 Too Many Requests>.") + self.__logger.warning(f"It is not safe to use more than {BfxWebSocketClient.MAXIMUM_CONNECTIONS_AMOUNT} " \ + f"buckets from the same connection ({connections} in use), the server could momentarily " \ + "block the client with <429 Too Many Requests>.") for _ in range(connections): - self.buckets += [BfxWebSocketBucket(self.host, self.event_emitter)] + _bucket = BfxWebSocketBucket( \ + self._host, self.__event_emitter) + + self.__buckets.update( { _bucket: None }) await self.__connect() - #pylint: disable-next=too-many-statements,too-many-branches - async def __connect(self): - Reconnection = namedtuple("Reconnection", ["status", "attempts", "timestamp"]) - reconnection = Reconnection(status=False, attempts=0, timestamp=None) - timer, tasks, on_timeout_event = None, [], asyncio.locks.Event() + #pylint: disable-next=too-many-branches + async def __connect(self) -> None: + class _Delay: + BACKOFF_MIN, BACKOFF_MAX = 1.92, 60.0 - delay = None + BACKOFF_INITIAL = 5.0 - def _on_wss_timeout(): - on_timeout_event.set() + def __init__(self, backoff_factor: float) -> None: + self.__backoff_factor = backoff_factor + self.__backoff_delay = _Delay.BACKOFF_MIN + self.__initial_delay = random.random() * _Delay.BACKOFF_INITIAL - #pylint: disable-next=too-many-branches - async def _connection(): - nonlocal reconnection, timer, tasks + def next(self) -> float: + _backoff_delay = self.peek() + __backoff_delay = self.__backoff_delay * self.__backoff_factor + self.__backoff_delay = min(__backoff_delay, _Delay.BACKOFF_MAX) - async with websockets.connect(self.host, ping_interval=None) as websocket: - if reconnection.status: - self.logger.info(f"Reconnection attempt successful (no.{reconnection.attempts}): The " \ - f"client has been offline for a total of {datetime.now() - reconnection.timestamp} " \ - f"(connection lost on: {reconnection.timestamp:%d-%m-%Y at %H:%M:%S}).") + return _backoff_delay - reconnection = Reconnection(status=False, attempts=0, timestamp=None) + def peek(self) -> float: + return (self.__backoff_delay == _Delay.BACKOFF_MIN) \ + and self.__initial_delay or self.__backoff_delay - if isinstance(timer, asyncio.events.TimerHandle): - timer.cancel() + def reset(self) -> None: + self.__backoff_delay = _Delay.BACKOFF_MIN - self.websocket = websocket + _delay = _Delay(backoff_factor=1.618) - coroutines = [ BfxWebSocketBucket.connect(bucket) for bucket in self.buckets ] + _on_wss_timeout = asyncio.locks.Event() - tasks = [ asyncio.create_task(coroutine) for coroutine in coroutines ] - - if len(self.buckets) == 0 or \ - (await asyncio.gather(*[bucket.wait() for bucket in self.buckets])): - self.event_emitter.emit("open") - - if self.credentials: - await self.__authenticate(**self.credentials) - - async for message in websocket: - message = json.loads(message) - - if isinstance(message, dict): - if message["event"] == "info" and "version" in message: - if BfxWebSocketClient.VERSION != message["version"]: - raise OutdatedClientVersion("Mismatch between the client version and the server " \ - "version. Update the library to the latest version to continue (client version: " \ - f"{BfxWebSocketClient.VERSION}, server version: {message['version']}).") - elif message["event"] == "info" and message["code"] == 20051: - rcvd = websockets.frames.Close(code=1012, - reason="Stop/Restart WebSocket Server (please reconnect).") - - raise websockets.exceptions.ConnectionClosedError(rcvd=rcvd, sent=None) - elif message["event"] == "auth": - if message["status"] != "OK": - raise InvalidAuthenticationCredentials( - "Cannot authenticate with given API-KEY and API-SECRET.") - - self.event_emitter.emit("authenticated", message) - - self.authentication = True - elif message["event"] == "error": - self.event_emitter.emit("wss-error", message["code"], message["msg"]) - - if isinstance(message, list): - if message[0] == 0 and message[1] != "hb": - self.handler.handle(message[1], message[2]) + def on_wss_timeout(): + if not self.open: + _on_wss_timeout.set() while True: - if reconnection.status: - await asyncio.sleep(delay.next()) + if self.__reconnection: + await asyncio.sleep(_delay.next()) - if on_timeout_event.is_set(): + if _on_wss_timeout.is_set(): raise ReconnectionTimeoutError("Connection has been offline for too long " \ - f"without being able to reconnect (wss_timeout: {self.wss_timeout}s).") + f"without being able to reconnect (wss_timeout: {self.__wss_timeout}s).") try: - await _connection() - except (websockets.exceptions.ConnectionClosedError, socket.gaierror) as error: - for task in tasks: - task.cancel() + await self.__connection() + except (websockets.exceptions.ConnectionClosedError, gaierror) as error: + for bucket in self.__buckets: + if (_task := self.__buckets[bucket]): + _task.cancel() if isinstance(error, websockets.exceptions.ConnectionClosedError) and error.code in (1006, 1012): if error.code == 1006: - self.logger.error("Connection lost: no close frame received " \ - "or sent (1006). Trying to reconnect...") + self.__logger.error("Connection lost: no close frame " \ + "received or sent (1006). Trying to reconnect...") if error.code == 1012: - self.logger.info("WSS server is about to restart, clients need " \ + self.__logger.info("WSS server is about to restart, clients need " \ "to reconnect (server sent 20051). Reconnection attempt in progress...") - reconnection = Reconnection(status=True, attempts=1, timestamp=datetime.now()) + if self.__wss_timeout is not None: + asyncio.get_event_loop().call_later( + self.__wss_timeout, on_wss_timeout) - if self.wss_timeout is not None: - timer = asyncio.get_event_loop().call_later(self.wss_timeout, _on_wss_timeout) + self.__reconnection = \ + { "attempts": 1, "reason": error.reason, "timestamp": datetime.now() } - delay = _Delay(backoff_factor=1.618) + self._authentication = False - self.authentication = False - elif isinstance(error, socket.gaierror) and reconnection.status: - self.logger.warning(f"Reconnection attempt was unsuccessful (no.{reconnection.attempts}). " \ - f"Next reconnection attempt in {delay.peek():.2f} seconds. (at the moment " \ - f"the client has been offline for {datetime.now() - reconnection.timestamp})") + _delay.reset() + elif isinstance(error, gaierror) and self.__reconnection: + self.__logger.warning( + f"_Reconnection attempt was unsuccessful (no.{self.__reconnection['attempts']}). " \ + f"Next reconnection attempt in {_delay.peek():.2f} seconds. (at the moment " \ + f"the client has been offline for {datetime.now() - self.__reconnection['timestamp']})") - reconnection = reconnection._replace(attempts=reconnection.attempts + 1) + self.__reconnection["attempts"] += 1 else: raise error - if not reconnection.status: - self.event_emitter.emit("disconnection", - self.websocket.close_code, self.websocket.close_reason) + if not self.__reconnection: + self.__event_emitter.emit("disconnection", + self._websocket.close_code, self._websocket.close_reason) break - async def __authenticate(self, api_key, api_secret, filters=None): - data = { "event": "auth", "filter": filters, "apiKey": api_key } + async def __connection(self) -> None: + async with _websockets__connect(self._host) as websocket: + if self.__reconnection: + self.__logger.info(f"_Reconnection attempt successful (no.{self.__reconnection['attempts']}): The " \ + f"client has been offline for a total of {datetime.now() - self.__reconnection['timestamp']} " \ + f"(connection lost on: {self.__reconnection['timestamp']:%d-%m-%Y at %H:%M:%S}).") - data["authNonce"] = str(round(time.time() * 1_000_000)) + self.__reconnection = None - data["authPayload"] = "AUTH" + data["authNonce"] + self._websocket = websocket - data["authSig"] = hmac.new( - api_secret.encode("utf8"), - data["authPayload"].encode("utf8"), - hashlib.sha384 - ).hexdigest() + self.__buckets = { + bucket: asyncio.create_task(_c) + for bucket in self.__buckets + if (_c := bucket.connect()) + } - await self.websocket.send(json.dumps(data)) + if len(self.__buckets) == 0 or \ + (await asyncio.gather(*[bucket.wait() for bucket in self.__buckets])): + self.__event_emitter.emit("open") - async def subscribe(self, channel, **kwargs): - if len(self.buckets) == 0: - raise ZeroConnectionsError("Unable to subscribe: the number of connections must be greater than 0.") + if self.__credentials: + authentication = BfxWebSocketClient. \ + __build_authentication_message(**self.__credentials) - counters = [ len(bucket.pendings) + len(bucket.subscriptions) for bucket in self.buckets ] + await self._websocket.send(authentication) + + async for message in self._websocket: + message = json.loads(message) + + if isinstance(message, dict): + if message["event"] == "info" and "version" in message: + if BfxWebSocketClient.VERSION != message["version"]: + raise OutdatedClientVersion("Mismatch between the client version and the server " \ + "version. Update the library to the latest version to continue (client version: " \ + f"{BfxWebSocketClient.VERSION}, server version: {message['version']}).") + elif message["event"] == "info" and message["code"] == 20051: + rcvd = websockets.frames.Close(code=1012, + reason="Stop/Restart WebSocket Server (please reconnect).") + + raise websockets.exceptions.ConnectionClosedError(rcvd=rcvd, sent=None) + elif message["event"] == "auth": + if message["status"] != "OK": + raise InvalidAuthenticationCredentials( + "Cannot authenticate with given API-KEY and API-SECRET.") + + self.__event_emitter.emit("authenticated", message) + + self._authentication = True + elif message["event"] == "error": + self.__event_emitter.emit("wss-error", message["code"], message["msg"]) + + if isinstance(message, list) and \ + message[0] == 0 and message[1] != Connection.HEARTBEAT: + self.__handler.handle(message[1], message[2]) + + @Connection.require_websocket_connection + async def subscribe(self, + channel: str, + sub_id: Optional[str] = None, + **kwargs: Any) -> None: + if len(self.__buckets) == 0: + raise ZeroConnectionsError("Unable to subscribe: " \ + "the number of connections must be greater than 0.") + + _buckets = list(self.__buckets.keys()) + + counters = [ len(bucket.pendings) + len(bucket.subscriptions) + for bucket in _buckets ] index = counters.index(min(counters)) - await self.buckets[index].subscribe(channel, **kwargs) + await _buckets[index] \ + .subscribe(channel, sub_id, **kwargs) - async def unsubscribe(self, sub_id): - for bucket in self.buckets: - if (chan_id := bucket.get_chan_id(sub_id)): - await bucket.unsubscribe(chan_id=chan_id) + @Connection.require_websocket_connection + async def unsubscribe(self, sub_id: str) -> None: + for bucket in self.__buckets: + if bucket.has(sub_id=sub_id): + await bucket.unsubscribe(sub_id=sub_id) - async def close(self, code=1000, reason=str()): - for bucket in self.buckets: + @Connection.require_websocket_connection + async def close(self, code: int = 1000, reason: str = str()) -> None: + for bucket in self.__buckets: await bucket.close(code=code, reason=reason) - if self.websocket is not None and self.websocket.open: - await self.websocket.close(code=code, reason=reason) + if self._websocket.open: + await self._websocket.close( \ + code=code, reason=reason) - @require_websocket_authentication - async def notify(self, info, message_id=None, **kwargs): - await self.websocket.send(json.dumps([ 0, "n", message_id, { "type": "ucm-test", "info": info, **kwargs } ])) + @Connection.Authenticable.require_websocket_authentication + async def notify(self, + info: Any, + message_id: Optional[int] = None, + **kwargs: Any) -> None: + await self._websocket.send( + json.dumps([ 0, "n", message_id, + { "type": "ucm-test", "info": info, **kwargs } ])) - @require_websocket_authentication - async def __handle_websocket_input(self, event, data): - await self.websocket.send(json.dumps([ 0, event, None, data], cls=JSONEncoder)) + @Connection.Authenticable.require_websocket_authentication + async def __handle_websocket_input(self, event: str, data: Any) -> None: + await self._websocket.send(json.dumps(\ + [ 0, event, None, data], cls=JSONEncoder)) - def on(self, *events, callback = None): + def on(self, *events: str, callback: Optional["_T"] = None) -> Callable[["_T"], None]: for event in events: if event not in BfxWebSocketClient.EVENTS: raise EventNotSupported(f"Event <{event}> is not supported. To get a list " \ - "of available events see BfxWebSocketClient.EVENTS.") + "of available events see BfxWebSocketClient.EVENTS.") - def _register_event(event, function): - if event in BfxWebSocketClient.__ONCE_EVENTS: - self.event_emitter.once(event, function) - else: self.event_emitter.on(event, function) - - if callback is not None: + def _register_events(function: "_T", events: Tuple[str, ...]) -> None: for event in events: - _register_event(event, callback) + if event in BfxWebSocketClient.__ONCE_EVENTS: + self.__event_emitter.once(event, function) + else: + self.__event_emitter.on(event, function) - if callback is None: - def handler(function): - for event in events: - _register_event(event, function) + if callback: + _register_events(callback, events) - return handler + def _handler(function: "_T") -> None: + _register_events(function, events) + + return _handler + + @staticmethod + def __build_authentication_message(api_key: str, + api_secret: str, + filters: Optional[List[str]] = None) -> str: + message: Dict[str, Any] = \ + { "event": "auth", "filter": filters, "apiKey": api_key } + + message["authNonce"] = round(datetime.now().timestamp() * 1_000_000) + + message["authPayload"] = f"AUTH{message['authNonce']}" + + message["authSig"] = hmac.new( + key=api_secret.encode("utf8"), + msg=message["authPayload"].encode("utf8"), + digestmod=hashlib.sha384 + ).hexdigest() + + return json.dumps(message) diff --git a/bfxapi/websocket/_connection.py b/bfxapi/websocket/_connection.py index 7908ca7..1562b43 100644 --- a/bfxapi/websocket/_connection.py +++ b/bfxapi/websocket/_connection.py @@ -1,39 +1,60 @@ -from typing import \ - TYPE_CHECKING, Optional, cast - -from bfxapi.websocket.exceptions import \ - ConnectionNotOpen - -if TYPE_CHECKING: - from websockets.client import WebSocketClientProtocol - -class Connection: - HEARTBEAT = "hb" - - def __init__(self, host: str) -> None: - self._host = host - - self.__protocol: Optional["WebSocketClientProtocol"] = None - - @property - def open(self) -> bool: - return self.__protocol is not None and \ - self.__protocol.open - - @property - def _websocket(self) -> "WebSocketClientProtocol": - return cast("WebSocketClientProtocol", self.__protocol) - - @_websocket.setter - def _websocket(self, protocol: "WebSocketClientProtocol") -> None: - self.__protocol = protocol - - @staticmethod - def require_websocket_connection(function): - async def wrapper(self, *args, **kwargs): - if self.open: - return await function(self, *args, **kwargs) - - raise ConnectionNotOpen("No open connection with the server.") - - return wrapper +from typing import \ + TYPE_CHECKING, Optional, cast + +from bfxapi.websocket.exceptions import \ + ConnectionNotOpen, ActionRequiresAuthentication + +if TYPE_CHECKING: + from websockets.client import WebSocketClientProtocol + +class Connection: + HEARTBEAT = "hb" + + class Authenticable: + def __init__(self) -> None: + self._authentication: bool = False + + @property + def authentication(self) -> bool: + return self._authentication + + @staticmethod + def require_websocket_authentication(function): + async def wrapper(self, *args, **kwargs): + if not self.authentication: + raise ActionRequiresAuthentication("To perform this action you need to " \ + "authenticate using your API_KEY and API_SECRET.") + + internal = Connection.require_websocket_connection(function) + + return await internal(self, *args, **kwargs) + + return wrapper + + def __init__(self, host: str) -> None: + self._host = host + + self.__protocol: Optional["WebSocketClientProtocol"] = None + + @property + def open(self) -> bool: + return self.__protocol is not None and \ + self.__protocol.open + + @property + def _websocket(self) -> "WebSocketClientProtocol": + return cast("WebSocketClientProtocol", self.__protocol) + + @_websocket.setter + def _websocket(self, protocol: "WebSocketClientProtocol") -> None: + self.__protocol = protocol + + @staticmethod + def require_websocket_connection(function): + async def wrapper(self, *args, **kwargs): + if self.open: + return await function(self, *args, **kwargs) + + raise ConnectionNotOpen("No open connection with the server.") + + return wrapper From 9edbd7a41558a83e78da718cb4dd4256b889e20e Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 19 Jun 2023 05:02:04 +0200 Subject: [PATCH 18/65] Rename bfxapi.utils to _utils (and update references). --- bfxapi/{utils => _utils}/__init__.py | 0 bfxapi/{utils => _utils}/json_encoder.py | 0 bfxapi/{utils => _utils}/logger.py | 0 bfxapi/rest/endpoints/rest_auth_endpoints.py | 2 +- bfxapi/rest/middleware/middleware.py | 2 +- bfxapi/types/dataclasses.py | 2 +- bfxapi/websocket/_client/bfx_websocket_client.py | 4 ++-- bfxapi/websocket/_client/bfx_websocket_inputs.py | 2 +- setup.py | 2 +- 9 files changed, 7 insertions(+), 7 deletions(-) rename bfxapi/{utils => _utils}/__init__.py (100%) rename bfxapi/{utils => _utils}/json_encoder.py (100%) rename bfxapi/{utils => _utils}/logger.py (100%) diff --git a/bfxapi/utils/__init__.py b/bfxapi/_utils/__init__.py similarity index 100% rename from bfxapi/utils/__init__.py rename to bfxapi/_utils/__init__.py diff --git a/bfxapi/utils/json_encoder.py b/bfxapi/_utils/json_encoder.py similarity index 100% rename from bfxapi/utils/json_encoder.py rename to bfxapi/_utils/json_encoder.py diff --git a/bfxapi/utils/logger.py b/bfxapi/_utils/logger.py similarity index 100% rename from bfxapi/utils/logger.py rename to bfxapi/_utils/logger.py diff --git a/bfxapi/rest/endpoints/rest_auth_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py index e1b9fab..ec4155e 100644 --- a/bfxapi/rest/endpoints/rest_auth_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -22,7 +22,7 @@ from ...types import serializers from ...types.serializers import _Notification -from ...utils.json_encoder import JSON +from ..._utils.json_encoder import JSON class RestAuthEndpoints(Middleware): def get_user_info(self) -> UserInfo: diff --git a/bfxapi/rest/middleware/middleware.py b/bfxapi/rest/middleware/middleware.py index 4bfe8b0..ae04b03 100644 --- a/bfxapi/rest/middleware/middleware.py +++ b/bfxapi/rest/middleware/middleware.py @@ -6,7 +6,7 @@ import time, hmac, hashlib, json, requests from ..enums import Error from ..exceptions import ResourceNotFound, RequestParametersError, InvalidAuthenticationCredentials, UnknownGenericError -from ...utils.json_encoder import JSONEncoder +from ..._utils.json_encoder import JSONEncoder if TYPE_CHECKING: from requests.sessions import _Params diff --git a/bfxapi/types/dataclasses.py b/bfxapi/types/dataclasses.py index 8b0b69f..d60cd38 100644 --- a/bfxapi/types/dataclasses.py +++ b/bfxapi/types/dataclasses.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from .labeler import _Type, partial, compose -from ..utils.json_encoder import JSON +from .._utils.json_encoder import JSON #region Dataclass definitions for types of public use diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 38a2c37..a0dc721 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -15,9 +15,9 @@ import \ from websockets.legacy.client import connect as _websockets__connect -from bfxapi.utils.json_encoder import JSONEncoder +from bfxapi._utils.json_encoder import JSONEncoder -from bfxapi.utils.logger import \ +from bfxapi._utils.logger import \ ColorLogger, FileLogger from bfxapi.websocket._handlers import \ diff --git a/bfxapi/websocket/_client/bfx_websocket_inputs.py b/bfxapi/websocket/_client/bfx_websocket_inputs.py index 71bf8a8..c84e9f5 100644 --- a/bfxapi/websocket/_client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/_client/bfx_websocket_inputs.py @@ -5,7 +5,7 @@ if TYPE_CHECKING: from bfxapi.enums import \ OrderType, FundingOfferType - from bfxapi.utils.json_encoder import JSON + from bfxapi._utils.json_encoder import JSON from decimal import Decimal diff --git a/setup.py b/setup.py index 8538ba9..9ca14a8 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( "Source": "https://github.com/bitfinexcom/bitfinex-api-py", }, packages=[ - "bfxapi", "bfxapi.utils", "bfxapi.types", + "bfxapi", "bfxapi._utils", "bfxapi.types", "bfxapi.websocket", "bfxapi.websocket._client", "bfxapi.websocket._handlers", "bfxapi.websocket._event_emitter", "bfxapi.rest", "bfxapi.rest.endpoints", "bfxapi.rest.middleware", From bae48b29012064ca4aaa453f83c67d03d11e77b2 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Tue, 20 Jun 2023 18:27:22 +0200 Subject: [PATCH 19/65] Improve wss_timeout implementation in BfxWebSocketClient. --- .../websocket/_client/bfx_websocket_client.py | 58 +++++++++++++------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index a0dc721..8ef5490 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -13,6 +13,8 @@ import \ hmac, hashlib, random, \ websockets +from websockets.exceptions import ConnectionClosedError + from websockets.legacy.client import connect as _websockets__connect from bfxapi._utils.json_encoder import JSONEncoder @@ -134,7 +136,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): await self.__connect() - #pylint: disable-next=too-many-branches + #pylint: disable-next=too-many-branches,too-many-statements async def __connect(self) -> None: class _Delay: BACKOFF_MIN, BACKOFF_MAX = 1.92, 60.0 @@ -162,28 +164,46 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): _delay = _Delay(backoff_factor=1.618) - _on_wss_timeout = asyncio.locks.Event() + _sleep: Optional["Task"] = None - def on_wss_timeout(): + def _on_wss_timeout(): if not self.open: - _on_wss_timeout.set() + if _sleep: + _sleep.cancel() while True: if self.__reconnection: - await asyncio.sleep(_delay.next()) + _sleep = asyncio.create_task( \ + asyncio.sleep(_delay.next())) - if _on_wss_timeout.is_set(): - raise ReconnectionTimeoutError("Connection has been offline for too long " \ - f"without being able to reconnect (wss_timeout: {self.__wss_timeout}s).") + try: + await _sleep + except asyncio.CancelledError: + raise ReconnectionTimeoutError("Connection has been offline for too long " \ + f"without being able to reconnect (wss_timeout: {self.__wss_timeout}s).") \ + from None try: await self.__connection() - except (websockets.exceptions.ConnectionClosedError, gaierror) as error: - for bucket in self.__buckets: - if (_task := self.__buckets[bucket]): - _task.cancel() + except (ConnectionClosedError, gaierror) as error: + async def _cancel(task: "Task") -> None: + task.cancel() - if isinstance(error, websockets.exceptions.ConnectionClosedError) and error.code in (1006, 1012): + try: + await task + except (ConnectionClosedError, gaierror) as _e: + if type(error) is not type(_e) or error.args != _e.args: + raise _e + except asyncio.CancelledError: + pass + + for bucket in self.__buckets: + if task := self.__buckets[bucket]: + self.__buckets[bucket] = None + + await _cancel(task) + + if isinstance(error, ConnectionClosedError) and error.code in (1006, 1012): if error.code == 1006: self.__logger.error("Connection lost: no close frame " \ "received or sent (1006). Trying to reconnect...") @@ -194,7 +214,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): if self.__wss_timeout is not None: asyncio.get_event_loop().call_later( - self.__wss_timeout, on_wss_timeout) + self.__wss_timeout, _on_wss_timeout) self.__reconnection = \ { "attempts": 1, "reason": error.reason, "timestamp": datetime.now() } @@ -214,7 +234,8 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): if not self.__reconnection: self.__event_emitter.emit("disconnection", - self._websocket.close_code, self._websocket.close_reason) + self._websocket.close_code, \ + self._websocket.close_reason) break @@ -255,10 +276,9 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): "version. Update the library to the latest version to continue (client version: " \ f"{BfxWebSocketClient.VERSION}, server version: {message['version']}).") elif message["event"] == "info" and message["code"] == 20051: - rcvd = websockets.frames.Close(code=1012, - reason="Stop/Restart WebSocket Server (please reconnect).") - - raise websockets.exceptions.ConnectionClosedError(rcvd=rcvd, sent=None) + code, reason = 1012, "Stop/Restart WebSocket Server (please reconnect)." + rcvd = websockets.frames.Close(code=code, reason=reason) + raise ConnectionClosedError(rcvd=rcvd, sent=None) elif message["event"] == "auth": if message["status"] != "OK": raise InvalidAuthenticationCredentials( From 755ee767a8f5388f50e48e1531741605ac64028a Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 23 Jun 2023 17:08:54 +0200 Subject: [PATCH 20/65] Improve bfxapi._utils.logger (and update usage in Client). --- bfxapi/_utils/logger.py | 86 +++++++++++-------- bfxapi/client.py | 18 ++-- .../websocket/_client/bfx_websocket_client.py | 55 +++++------- 3 files changed, 85 insertions(+), 74 deletions(-) diff --git a/bfxapi/_utils/logger.py b/bfxapi/_utils/logger.py index 6ebac5a..df9c807 100644 --- a/bfxapi/_utils/logger.py +++ b/bfxapi/_utils/logger.py @@ -1,51 +1,67 @@ -import logging, sys +from typing import \ + TYPE_CHECKING, Literal, Optional -BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) +#pylint: disable-next=wildcard-import,unused-wildcard-import +from logging import * -COLOR_SEQ, ITALIC_COLOR_SEQ = "\033[1;%dm", "\033[3;%dm" +from copy import copy -COLORS = { - "DEBUG": CYAN, - "INFO": BLUE, - "WARNING": YELLOW, - "ERROR": RED -} +import sys -RESET_SEQ = "\033[0m" +if TYPE_CHECKING: + _Level = Literal["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] -class _ColorFormatter(logging.Formatter): - def __init__(self, msg, use_color = True): - logging.Formatter.__init__(self, msg, "%d-%m-%Y %H:%M:%S") +_BLACK, _RED, _GREEN, _YELLOW, \ +_BLUE, _MAGENTA, _CYAN, _WHITE = \ + [ f"\033[0;{90 + i}m" for i in range(8) ] - self.use_color = use_color +_BOLD_BLACK, _BOLD_RED, _BOLD_GREEN, _BOLD_YELLOW, \ +_BOLD_BLUE, _BOLD_MAGENTA, _BOLD_CYAN, _BOLD_WHITE = \ + [ f"\033[1;{90 + i}m" for i in range(8) ] - def format(self, record): - levelname = record.levelname - if self.use_color and levelname in COLORS: - record.name = ITALIC_COLOR_SEQ % (30 + BLACK) + record.name + RESET_SEQ - record.levelname = COLOR_SEQ % (30 + COLORS[levelname]) + levelname + RESET_SEQ - return logging.Formatter.format(self, record) +_NC = "\033[0m" -class ColorLogger(logging.Logger): - FORMAT = "[%(name)s] [%(levelname)s] [%(asctime)s] %(message)s" +class _ColorFormatter(Formatter): + __LEVELS = { + "INFO": _BLUE, + "WARNING": _YELLOW, + "ERROR": _RED, + "CRITICAL": _BOLD_RED, + "DEBUG": _BOLD_WHITE + } - def __init__(self, name, level): - logging.Logger.__init__(self, name, level) + def format(self, record: LogRecord) -> str: + _record = copy(record) + _record.name = _MAGENTA + record.name + _NC + _record.levelname = _ColorFormatter.__format_level(record.levelname) - colored_formatter = _ColorFormatter(self.FORMAT, use_color=True) - handler = logging.StreamHandler(stream=sys.stderr) - handler.setFormatter(fmt=colored_formatter) + return super().format(_record) - self.addHandler(hdlr=handler) + #pylint: disable-next=invalid-name + def formatTime(self, record: LogRecord, datefmt: Optional[str] = None) -> str: + return _GREEN + super().formatTime(record, datefmt) + _NC -class FileLogger(logging.Logger): - FORMAT = "[%(name)s] [%(levelname)s] [%(asctime)s] %(message)s" + @staticmethod + def __format_level(level: str) -> str: + return _ColorFormatter.__LEVELS[level] + level + _NC - def __init__(self, name, level, filename): - logging.Logger.__init__(self, name, level) +_FORMAT = "%(asctime)s %(name)s %(levelname)s %(message)s" - formatter = logging.Formatter(self.FORMAT) - handler = logging.FileHandler(filename=filename) +_DATE_FORMAT = "%d-%m-%Y %H:%M:%S" + +class ColorLogger(Logger): + __FORMATTER = Formatter(_FORMAT,_DATE_FORMAT) + + def __init__(self, name: str, level: "_Level" = "NOTSET") -> None: + super().__init__(name, level) + + formatter = _ColorFormatter(_FORMAT, _DATE_FORMAT) + + handler = StreamHandler(stream=sys.stderr) handler.setFormatter(fmt=formatter) - + self.addHandler(hdlr=handler) + + def register(self, filename: str) -> None: + handler = FileHandler(filename=filename) + handler.setFormatter(fmt=ColorLogger.__FORMATTER) self.addHandler(hdlr=handler) diff --git a/bfxapi/client.py b/bfxapi/client.py index 5e5292d..4a7bbdf 100644 --- a/bfxapi/client.py +++ b/bfxapi/client.py @@ -1,8 +1,10 @@ from typing import List, Literal, Optional -from .rest import BfxRestInterface -from .websocket import BfxWebSocketClient -from .urls import REST_HOST, WSS_HOST +from bfxapi._utils.logger import ColorLogger + +from bfxapi.rest import BfxRestInterface +from bfxapi.websocket import BfxWebSocketClient +from bfxapi.urls import REST_HOST, WSS_HOST class Client: def __init__( @@ -15,10 +17,14 @@ class Client: filters: Optional[List[str]] = None, wss_timeout: Optional[float] = 60 * 15, log_filename: Optional[str] = None, - log_level: Literal["ERROR", "WARNING", "INFO", "DEBUG"] = "INFO" + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" ) -> None: + logger = ColorLogger("bfxapi", level=log_level) + + if log_filename: + logger.register(filename=log_filename) + self.rest = BfxRestInterface(rest_host, api_key, api_secret) self.wss = BfxWebSocketClient(wss_host, api_key, api_secret, - filters=filters, wss_timeout=wss_timeout, log_filename=log_filename, - log_level=log_level) + filters=filters, wss_timeout=wss_timeout, logger=logger) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 8ef5490..8ff481f 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -1,8 +1,9 @@ from typing import \ TYPE_CHECKING, TypeVar, TypedDict,\ - Callable, Optional, Literal,\ - Tuple, List, Dict, \ - Any + Callable, Optional, Tuple, \ + List, Dict, Any + +from logging import Logger from datetime import datetime @@ -19,9 +20,6 @@ from websockets.legacy.client import connect as _websockets__connect from bfxapi._utils.json_encoder import JSONEncoder -from bfxapi._utils.logger import \ - ColorLogger, FileLogger - from bfxapi.websocket._handlers import \ PublicChannelsHandler, AuthEventsHandler @@ -38,8 +36,6 @@ from .bfx_websocket_bucket import BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs if TYPE_CHECKING: - from logging import Logger - from asyncio import Task _T = TypeVar("_T", bound=Callable[..., None]) @@ -50,6 +46,8 @@ if TYPE_CHECKING: _Reconnection = TypedDict("_Reconnection", { "attempts": int, "reason": str, "timestamp": datetime }) +_DEFAULT_LOGGER = Logger("bfxapi.websocket._client", level=0) + class BfxWebSocketClient(Connection, Connection.Authenticable): VERSION = BfxWebSocketBucket.VERSION @@ -69,22 +67,14 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): def __init__(self, host: str, - api_key: Optional[str] = None, - api_secret: Optional[str] = None, *, - filters: Optional[List[str]] = None, - wss_timeout: Optional[float] = 60 * 15, - log_filename: Optional[str] = None, - log_level: Literal["ERROR", "WARNING", "INFO", "DEBUG"] = "INFO") -> None: + credentials: Optional["_Credentials"] = None, + timeout: Optional[float] = 60 * 15, + logger: Logger = _DEFAULT_LOGGER) -> None: super().__init__(host) - self.__credentials: Optional["_Credentials"] = None - - if api_key and api_secret: - self.__credentials = \ - { "api_key": api_key, "api_secret": api_secret, "filters": filters } - - self.__wss_timeout = wss_timeout + self.__credentials, self.__timeout, self.__logger = \ + credentials, timeout, logger self.__event_emitter = BfxEventEmitter(targets = \ PublicChannelsHandler.ONCE_PER_SUBSCRIPTION + \ @@ -100,16 +90,15 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): self.__reconnection: Optional[_Reconnection] = None - self.__logger: "Logger" + @self.__event_emitter.on("error") + def error(exception: Exception) -> None: + header = f"{type(exception).__name__}: {str(exception)}" - if log_filename is None: - self.__logger = ColorLogger("BfxWebSocketClient", level=log_level) - else: self.__logger = FileLogger("BfxWebSocketClient", level=log_level, filename=log_filename) + stack_trace = traceback.format_exception( \ + type(exception), exception, exception.__traceback__) - self.__event_emitter.add_listener("error", - lambda exception: self.__logger.error(f"{type(exception).__name__}: {str(exception)}" + "\n" + - str().join(traceback.format_exception(type(exception), exception, exception.__traceback__))[:-1]) - ) + self.__logger.critical( \ + header + "\n" + str().join(stack_trace)[:-1]) @property def inputs(self) -> BfxWebSocketInputs: @@ -166,7 +155,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): _sleep: Optional["Task"] = None - def _on_wss_timeout(): + def _on_timeout(): if not self.open: if _sleep: _sleep.cancel() @@ -180,7 +169,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): await _sleep except asyncio.CancelledError: raise ReconnectionTimeoutError("Connection has been offline for too long " \ - f"without being able to reconnect (wss_timeout: {self.__wss_timeout}s).") \ + f"without being able to reconnect (timeout: {self.__timeout}s).") \ from None try: @@ -212,9 +201,9 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): self.__logger.info("WSS server is about to restart, clients need " \ "to reconnect (server sent 20051). Reconnection attempt in progress...") - if self.__wss_timeout is not None: + if self.__timeout is not None: asyncio.get_event_loop().call_later( - self.__wss_timeout, _on_wss_timeout) + self.__timeout, _on_timeout) self.__reconnection = \ { "attempts": 1, "reason": error.reason, "timestamp": datetime.now() } From faffb7fe82455e56f93b8e7c986f9342da264c53 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 23 Jun 2023 17:27:25 +0200 Subject: [PATCH 21/65] Add and implement new IncompleteCredentialError in bfxapi.client. --- bfxapi/client.py | 31 +++++++++++++++---- bfxapi/exceptions.py | 5 +++ .../websocket/_client/bfx_websocket_client.py | 5 ++- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/bfxapi/client.py b/bfxapi/client.py index 4a7bbdf..fe86615 100644 --- a/bfxapi/client.py +++ b/bfxapi/client.py @@ -1,11 +1,18 @@ -from typing import List, Literal, Optional +from typing import \ + TYPE_CHECKING, TypedDict, List, Literal, Optional from bfxapi._utils.logger import ColorLogger +from bfxapi.exceptions import IncompleteCredentialError + from bfxapi.rest import BfxRestInterface from bfxapi.websocket import BfxWebSocketClient from bfxapi.urls import REST_HOST, WSS_HOST +if TYPE_CHECKING: + _Credentials = TypedDict("_Credentials", \ + { "api_key": str, "api_secret": str, "filters": Optional[List[str]] }) + class Client: def __init__( self, @@ -15,16 +22,28 @@ class Client: rest_host: str = REST_HOST, wss_host: str = WSS_HOST, filters: Optional[List[str]] = None, - wss_timeout: Optional[float] = 60 * 15, + timeout: Optional[float] = 60 * 15, log_filename: Optional[str] = None, log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" ) -> None: + credentials: Optional["_Credentials"] = None + + if api_key and api_secret: + credentials = \ + { "api_key": api_key, "api_secret": api_secret, "filters": filters } + elif api_key: + raise IncompleteCredentialError( \ + "You must provide both an API-KEY and an API-SECRET (missing API-KEY).") + elif api_secret: + raise IncompleteCredentialError( \ + "You must provide both an API-KEY and an API-SECRET (missing API-SECRET).") + + self.rest = BfxRestInterface(rest_host, api_key, api_secret) + logger = ColorLogger("bfxapi", level=log_level) if log_filename: logger.register(filename=log_filename) - self.rest = BfxRestInterface(rest_host, api_key, api_secret) - - self.wss = BfxWebSocketClient(wss_host, api_key, api_secret, - filters=filters, wss_timeout=wss_timeout, logger=logger) + self.wss = BfxWebSocketClient(wss_host, \ + credentials=credentials, timeout=timeout, logger=logger) diff --git a/bfxapi/exceptions.py b/bfxapi/exceptions.py index 136f5f1..b636119 100644 --- a/bfxapi/exceptions.py +++ b/bfxapi/exceptions.py @@ -6,3 +6,8 @@ class BfxBaseException(Exception): """ Base class for every custom exception in bfxapi/rest/exceptions.py and bfxapi/websocket/exceptions.py. """ + +class IncompleteCredentialError(BfxBaseException): + """ + This error indicates an incomplete credential object (missing api-key or api-secret). + """ diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 8ff481f..dd02469 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -36,13 +36,12 @@ from .bfx_websocket_bucket import BfxWebSocketBucket from .bfx_websocket_inputs import BfxWebSocketInputs if TYPE_CHECKING: + from bfxapi.client import _Credentials + from asyncio import Task _T = TypeVar("_T", bound=Callable[..., None]) - _Credentials = TypedDict("_Credentials", \ - { "api_key": str, "api_secret": str, "filters": Optional[List[str]] }) - _Reconnection = TypedDict("_Reconnection", { "attempts": int, "reason": str, "timestamp": datetime }) From 4ba6b28f5b7da00160b58b991fe2206140945480 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 23 Jun 2023 17:53:05 +0200 Subject: [PATCH 22/65] Rename bfxapi._utils.logger to bfxapi._utils.logging (and update references). --- bfxapi/_utils/{logger.py => logging.py} | 0 bfxapi/client.py | 2 +- bfxapi/websocket/_client/bfx_websocket_client.py | 12 ++++++------ 3 files changed, 7 insertions(+), 7 deletions(-) rename bfxapi/_utils/{logger.py => logging.py} (100%) diff --git a/bfxapi/_utils/logger.py b/bfxapi/_utils/logging.py similarity index 100% rename from bfxapi/_utils/logger.py rename to bfxapi/_utils/logging.py diff --git a/bfxapi/client.py b/bfxapi/client.py index fe86615..dd9127c 100644 --- a/bfxapi/client.py +++ b/bfxapi/client.py @@ -1,7 +1,7 @@ from typing import \ TYPE_CHECKING, TypedDict, List, Literal, Optional -from bfxapi._utils.logger import ColorLogger +from bfxapi._utils.logging import ColorLogger from bfxapi.exceptions import IncompleteCredentialError diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index dd02469..85abb85 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -75,20 +75,20 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): self.__credentials, self.__timeout, self.__logger = \ credentials, timeout, logger + self.__buckets: Dict[BfxWebSocketBucket, Optional["Task"]] = { } + + self.__reconnection: Optional[_Reconnection] = None + self.__event_emitter = BfxEventEmitter(targets = \ PublicChannelsHandler.ONCE_PER_SUBSCRIPTION + \ ["subscribed"]) - self.__handler = AuthEventsHandler(\ + self.__handler = AuthEventsHandler( \ event_emitter=self.__event_emitter) - self.__inputs = BfxWebSocketInputs(\ + self.__inputs = BfxWebSocketInputs( \ handle_websocket_input=self.__handle_websocket_input) - self.__buckets: Dict[BfxWebSocketBucket, Optional["Task"]] = { } - - self.__reconnection: Optional[_Reconnection] = None - @self.__event_emitter.on("error") def error(exception: Exception) -> None: header = f"{type(exception).__name__}: {str(exception)}" From da2b411265dee7eff63b74c1e93a9cb486b2afef Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Tue, 25 Jul 2023 16:05:12 +0200 Subject: [PATCH 23/65] Fix missing return statement in public_channels_handler.__raw_book_channel_handler. --- bfxapi/websocket/_handlers/public_channels_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index 87c46e4..004416d 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -108,7 +108,7 @@ class PublicChannelsHandler: def __raw_book_channel_handler(self, subscription: "Book", stream: List[Any]): if subscription["symbol"].startswith("t"): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): - self.__event_emitter.emit("t_raw_book_snapshot", subscription, \ + return self.__event_emitter.emit("t_raw_book_snapshot", subscription, \ [ serializers.TradingPairRawBook.parse(*sub_stream) \ for sub_stream in stream[0] ]) From 3038027f35cef1bd7ce5025dbdb4741db05b4c6e Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Tue, 25 Jul 2023 16:21:14 +0200 Subject: [PATCH 24/65] Add fix to handle InvalidStatusCode exception (for 408 Request Timeout). --- .../websocket/_client/bfx_websocket_client.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 85abb85..2031406 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -4,9 +4,7 @@ from typing import \ List, Dict, Any from logging import Logger - from datetime import datetime - from socket import gaierror import \ @@ -14,22 +12,27 @@ import \ hmac, hashlib, random, \ websockets -from websockets.exceptions import ConnectionClosedError +from websockets.exceptions import \ + ConnectionClosedError, \ + InvalidStatusCode -from websockets.legacy.client import connect as _websockets__connect +from websockets.legacy.client import \ + connect as _websockets__connect from bfxapi._utils.json_encoder import JSONEncoder - -from bfxapi.websocket._handlers import \ - PublicChannelsHandler, AuthEventsHandler - from bfxapi.websocket._connection import Connection - from bfxapi.websocket._event_emitter import BfxEventEmitter +from bfxapi.websocket._handlers import \ + PublicChannelsHandler, \ + AuthEventsHandler + from bfxapi.websocket.exceptions import \ - InvalidAuthenticationCredentials, EventNotSupported, ZeroConnectionsError, \ - ReconnectionTimeoutError, OutdatedClientVersion + InvalidAuthenticationCredentials, \ + ReconnectionTimeoutError, \ + OutdatedClientVersion, \ + ZeroConnectionsError, \ + EventNotSupported from .bfx_websocket_bucket import BfxWebSocketBucket @@ -173,7 +176,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): try: await self.__connection() - except (ConnectionClosedError, gaierror) as error: + except (ConnectionClosedError, InvalidStatusCode, gaierror) as error: async def _cancel(task: "Task") -> None: task.cancel() @@ -210,7 +213,8 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): self._authentication = False _delay.reset() - elif isinstance(error, gaierror) and self.__reconnection: + elif ((isinstance(error, InvalidStatusCode) and error.status_code == 408) or \ + isinstance(error, gaierror)) and self.__reconnection: self.__logger.warning( f"_Reconnection attempt was unsuccessful (no.{self.__reconnection['attempts']}). " \ f"Next reconnection attempt in {_delay.peek():.2f} seconds. (at the moment " \ From d9267de00983c67569f1115d67c3e7ed49a6bbde Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 14:57:35 +0200 Subject: [PATCH 25/65] Add config to enable checksums in BfxWebSocketBucket. --- bfxapi/websocket/_client/bfx_websocket_bucket.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index c8dcf97..07c5034 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -14,6 +14,8 @@ if TYPE_CHECKING: from websockets.client import WebSocketClientProtocol from pyee import EventEmitter +_CHECKSUM_FLAG_VALUE = 131_072 + class BfxWebSocketBucket(Connection): VERSION = 2 @@ -45,6 +47,8 @@ class BfxWebSocketBucket(Connection): await self.__recover_state() + await self.__set_conf(flags=_CHECKSUM_FLAG_VALUE) + async with self.__condition: self.__condition.notify(1) @@ -84,6 +88,10 @@ class BfxWebSocketBucket(Connection): self.__subscriptions.clear() + async def __set_conf(self, flags: int) -> None: + await self._websocket.send(json.dumps( \ + { "event": "conf", "flags": flags })) + @Connection.require_websocket_connection async def subscribe(self, channel: str, From 3c02232f425d4d52f8e742a9da4a004a48755edd Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 15:07:11 +0200 Subject: [PATCH 26/65] Add event handler for checksum messages (PublicChannelsHandler). --- .../_handlers/public_channels_handler.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index 004416d..bfc7208 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -12,6 +12,8 @@ if TYPE_CHECKING: _NoHeaderSubscription = \ Union[Ticker, Trades, Book, Candles, Status] +_CHECKSUM = "cs" + class PublicChannelsHandler: ONCE_PER_SUBSCRIPTION = [ "t_trades_snapshot", "f_trades_snapshot", "t_book_snapshot", @@ -25,7 +27,9 @@ class PublicChannelsHandler: "t_trade_execution_update", "f_trade_execution", "f_trade_execution_update", "t_book_update", "f_book_update", "t_raw_book_update", "f_raw_book_update", "candles_update", "derivatives_status_update", - "liquidation_feed_update" + "liquidation_feed_update", + + "checksum" ] def __init__(self, event_emitter: "EventEmitter") -> None: @@ -45,10 +49,13 @@ class PublicChannelsHandler: elif subscription["channel"] == "book": _subscription = cast("Book", _subscription) - if _subscription["prec"] != "R0": - self.__book_channel_handler(_subscription, stream) + if stream[0] == _CHECKSUM: + self.__checksum_handler(_subscription, stream[1]) else: - self.__raw_book_channel_handler(_subscription, stream) + if _subscription["prec"] != "R0": + self.__book_channel_handler(_subscription, stream) + else: + self.__raw_book_channel_handler(_subscription, stream) elif subscription["channel"] == "candles": self.__candles_channel_handler(cast("Candles", _subscription), stream) elif subscription["channel"] == "status": @@ -141,3 +148,6 @@ class PublicChannelsHandler: if subscription["key"].startswith("liq:"): return self.__event_emitter.emit("liquidation_feed_update", subscription, \ serializers.Liquidation.parse(*stream[0][0])) + + def __checksum_handler(self, subscription: "Book", value: int): + return self.__event_emitter.emit("checksum", subscription, value) From ce23a8991a0fe38982361efd1ee4f5b673181be5 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 15:19:44 +0200 Subject: [PATCH 27/65] Block negative checksums for possible race condition (PublicChannelsHandler::__checksum_handler). --- bfxapi/websocket/_handlers/public_channels_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index bfc7208..fa9b838 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -150,4 +150,6 @@ class PublicChannelsHandler: serializers.Liquidation.parse(*stream[0][0])) def __checksum_handler(self, subscription: "Book", value: int): - return self.__event_emitter.emit("checksum", subscription, value) + if not value < 0: + return self.__event_emitter.emit( \ + "checksum", subscription, value) From 26f25e5848d3ba5b351677e5bd8dc9fff464d684 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 16:41:52 +0200 Subject: [PATCH 28/65] Fix bug in method BfxWebSocketBucket::unsubscribe. --- bfxapi/websocket/_client/bfx_websocket_bucket.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 07c5034..c62a436 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -113,12 +113,11 @@ class BfxWebSocketBucket(Connection): @Connection.require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: - for subscription in self.__subscriptions.values(): + for chan_id, subscription in self.__subscriptions.items(): if subscription["subId"] == sub_id: - data = { "event": "unsubscribe", \ - "chanId": subscription["subId"] } - - message = json.dumps(data) + message = json.dumps({ + "event": "unsubscribe", + "chanId": chan_id }) await self._websocket.send(message) From f39b054397f988c931a19939359c3c46eb29bc81 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 17:29:41 +0200 Subject: [PATCH 29/65] Add implementation for BfxWebSocketClient::resubscribe and BfxWebSocketBucket::resubscribe. --- bfxapi/websocket/_client/bfx_websocket_bucket.py | 15 ++++++++++++++- bfxapi/websocket/_client/bfx_websocket_client.py | 6 ++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index c62a436..0130dfb 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -84,7 +84,8 @@ class BfxWebSocketBucket(Connection): _subscription = cast(Dict[str, Any], subscription) await self.subscribe( \ - sub_id=_subscription.pop("subId"), **_subscription) + sub_id=_subscription.pop("subId"), + **_subscription) self.__subscriptions.clear() @@ -121,6 +122,18 @@ class BfxWebSocketBucket(Connection): await self._websocket.send(message) + @Connection.require_websocket_connection + async def resubscribe(self, sub_id: str) -> None: + for subscription in self.__subscriptions.values(): + if subscription["subId"] == sub_id: + _subscription = cast(Dict[str, Any], subscription) + + await self.unsubscribe(sub_id=sub_id) + + await self.subscribe( \ + sub_id=_subscription.pop("subId"), + **_subscription) + @Connection.require_websocket_connection async def close(self, code: int = 1000, reason: str = str()) -> None: await self._websocket.close(code=code, reason=reason) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 2031406..178268c 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -311,6 +311,12 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): if bucket.has(sub_id=sub_id): await bucket.unsubscribe(sub_id=sub_id) + @Connection.require_websocket_connection + async def resubscribe(self, sub_id: str) -> None: + for bucket in self.__buckets: + if bucket.has(sub_id=sub_id): + await bucket.resubscribe(sub_id=sub_id) + @Connection.require_websocket_connection async def close(self, code: int = 1000, reason: str = str()) -> None: for bucket in self.__buckets: From 3a06b22247079e275ed390a54509edec4f30b61d Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 17:29:51 +0200 Subject: [PATCH 30/65] Add order book checksum handling in /examples/websocket/public/order_book.py. --- examples/websocket/public/order_book.py | 48 +++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/examples/websocket/public/order_book.py b/examples/websocket/public/order_book.py index ef6e31c..185ca56 100644 --- a/examples/websocket/public/order_book.py +++ b/examples/websocket/public/order_book.py @@ -2,7 +2,9 @@ from collections import OrderedDict -from typing import List +from typing import List, Dict + +import crcmod from bfxapi import Client, PUB_WSS_HOST @@ -14,10 +16,13 @@ class OrderBook: def __init__(self, symbols: List[str]): self.__order_book = { symbol: { - "bids": OrderedDict(), "asks": OrderedDict() + "bids": OrderedDict(), "asks": OrderedDict() } for symbol in symbols } + self.cooldown: Dict[str, bool] = \ + { symbol: False for symbol in symbols } + def update(self, symbol: str, data: TradingPairBook) -> None: price, count, amount = data.price, data.count, data.amount @@ -34,6 +39,31 @@ class OrderBook: if price in self.__order_book[symbol][kind]: del self.__order_book[symbol][kind][price] + def verify(self, symbol: str, checksum: int) -> bool: + values: List[int] = [ ] + + bids = sorted([ (data["price"], data["count"], data["amount"]) \ + for _, data in self.__order_book[symbol]["bids"].items() ], + key=lambda data: -data[0]) + + asks = sorted([ (data["price"], data["count"], data["amount"]) \ + for _, data in self.__order_book[symbol]["asks"].items() ], + key=lambda data: data[0]) + + if len(bids) < 25 or len(asks) < 25: + raise AssertionError("Not enough bids or asks (need at least 25).") + + for _i in range(25): + bid, ask = bids[_i], asks[_i] + values.extend([ bid[0], bid[2] ]) + values.extend([ ask[0], ask[2] ]) + + local = ":".join(str(value) for value in values).encode("UTF-8") + + crc32 = crcmod.mkCrcFun(0x104C11DB7, initCrc=0, xorOut=0xFFFFFFFF) + + return crc32(local) == checksum + SYMBOLS = [ "tBTCUSD", "tLTCUSD", "tLTCBTC", "tETHUSD", "tETHBTC" ] order_book = OrderBook(symbols=SYMBOLS) @@ -62,4 +92,18 @@ def on_t_book_snapshot(subscription: Book, snapshot: List[TradingPairBook]): def on_t_book_update(subscription: Book, data: TradingPairBook): order_book.update(subscription["symbol"], data) +@bfx.wss.on("checksum") +async def on_checksum(subscription: Book, value: int): + symbol = subscription["symbol"] + + if order_book.verify(symbol, value): + order_book.cooldown[symbol] = False + elif not order_book.cooldown[symbol]: + print("Mismatch between local and remote checksums: " + f"restarting book for symbol <{symbol}>...") + + await bfx.wss.resubscribe(sub_id=subscription["subId"]) + + order_book.cooldown[symbol] = True + bfx.wss.run() From 875d1d61e05191308547d7d10cea2f35e1f128f0 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 17:37:15 +0200 Subject: [PATCH 31/65] Add order book checksum handling in /examples/websocket/public/raw_order_book.py. --- examples/websocket/public/raw_order_book.py | 46 ++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/examples/websocket/public/raw_order_book.py b/examples/websocket/public/raw_order_book.py index 33ef321..264c3f0 100644 --- a/examples/websocket/public/raw_order_book.py +++ b/examples/websocket/public/raw_order_book.py @@ -2,7 +2,9 @@ from collections import OrderedDict -from typing import List +from typing import List, Dict + +import crcmod from bfxapi import Client, PUB_WSS_HOST @@ -18,6 +20,9 @@ class RawOrderBook: } for symbol in symbols } + self.cooldown: Dict[str, bool] = \ + { symbol: False for symbol in symbols } + def update(self, symbol: str, data: TradingPairRawBook) -> None: order_id, price, amount = data.order_id, data.price, data.amount @@ -34,6 +39,31 @@ class RawOrderBook: if order_id in self.__raw_order_book[symbol][kind]: del self.__raw_order_book[symbol][kind][order_id] + def verify(self, symbol: str, checksum: int) -> bool: + values: List[int] = [ ] + + bids = sorted([ (data["order_id"], data["price"], data["amount"]) \ + for _, data in self.__raw_order_book[symbol]["bids"].items() ], + key=lambda data: (-data[1], data[0])) + + asks = sorted([ (data["order_id"], data["price"], data["amount"]) \ + for _, data in self.__raw_order_book[symbol]["asks"].items() ], + key=lambda data: (data[1], data[0])) + + if len(bids) < 25 or len(asks) < 25: + raise AssertionError("Not enough bids or asks (need at least 25).") + + for _i in range(25): + bid, ask = bids[_i], asks[_i] + values.extend([ bid[0], bid[2] ]) + values.extend([ ask[0], ask[2] ]) + + local = ":".join(str(value) for value in values).encode("UTF-8") + + crc32 = crcmod.mkCrcFun(0x104C11DB7, initCrc=0, xorOut=0xFFFFFFFF) + + return crc32(local) == checksum + SYMBOLS = [ "tBTCUSD", "tLTCUSD", "tLTCBTC", "tETHUSD", "tETHBTC" ] raw_order_book = RawOrderBook(symbols=SYMBOLS) @@ -62,4 +92,18 @@ def on_t_raw_book_snapshot(subscription: Book, snapshot: List[TradingPairRawBook def on_t_raw_book_update(subscription: Book, data: TradingPairRawBook): raw_order_book.update(subscription["symbol"], data) +@bfx.wss.on("checksum") +async def on_checksum(subscription: Book, value: int): + symbol = subscription["symbol"] + + if raw_order_book.verify(symbol, value): + raw_order_book.cooldown[symbol] = False + elif not raw_order_book.cooldown[symbol]: + print("Mismatch between local and remote checksums: " + f"restarting book for symbol <{symbol}>...") + + await bfx.wss.resubscribe(sub_id=subscription["subId"]) + + raw_order_book.cooldown[symbol] = True + bfx.wss.run() From f6c49f677d0a4889f2564ed82626218c9c754843 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 28 Jul 2023 17:53:39 +0200 Subject: [PATCH 32/65] Remove block for negative checksums (and replace crcmod with native zlip module). --- bfxapi/websocket/_handlers/public_channels_handler.py | 5 ++--- examples/websocket/public/order_book.py | 10 +++++----- examples/websocket/public/raw_order_book.py | 10 +++++----- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index fa9b838..0bcd805 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -150,6 +150,5 @@ class PublicChannelsHandler: serializers.Liquidation.parse(*stream[0][0])) def __checksum_handler(self, subscription: "Book", value: int): - if not value < 0: - return self.__event_emitter.emit( \ - "checksum", subscription, value) + return self.__event_emitter.emit( \ + "checksum", subscription, value & 0xFFFFFFFF) diff --git a/examples/websocket/public/order_book.py b/examples/websocket/public/order_book.py index 185ca56..03d8c02 100644 --- a/examples/websocket/public/order_book.py +++ b/examples/websocket/public/order_book.py @@ -4,7 +4,7 @@ from collections import OrderedDict from typing import List, Dict -import crcmod +import zlib from bfxapi import Client, PUB_WSS_HOST @@ -58,13 +58,13 @@ class OrderBook: values.extend([ bid[0], bid[2] ]) values.extend([ ask[0], ask[2] ]) - local = ":".join(str(value) for value in values).encode("UTF-8") + local = ":".join(str(value) for value in values) - crc32 = crcmod.mkCrcFun(0x104C11DB7, initCrc=0, xorOut=0xFFFFFFFF) + crc32 = zlib.crc32(local.encode("UTF-8")) - return crc32(local) == checksum + return crc32 == checksum -SYMBOLS = [ "tBTCUSD", "tLTCUSD", "tLTCBTC", "tETHUSD", "tETHBTC" ] +SYMBOLS = [ "tLTCBTC", "tETHUSD", "tETHBTC" ] order_book = OrderBook(symbols=SYMBOLS) diff --git a/examples/websocket/public/raw_order_book.py b/examples/websocket/public/raw_order_book.py index 264c3f0..c151dda 100644 --- a/examples/websocket/public/raw_order_book.py +++ b/examples/websocket/public/raw_order_book.py @@ -4,7 +4,7 @@ from collections import OrderedDict from typing import List, Dict -import crcmod +import zlib from bfxapi import Client, PUB_WSS_HOST @@ -58,13 +58,13 @@ class RawOrderBook: values.extend([ bid[0], bid[2] ]) values.extend([ ask[0], ask[2] ]) - local = ":".join(str(value) for value in values).encode("UTF-8") + local = ":".join(str(value) for value in values) - crc32 = crcmod.mkCrcFun(0x104C11DB7, initCrc=0, xorOut=0xFFFFFFFF) + crc32 = zlib.crc32(local.encode("UTF-8")) - return crc32(local) == checksum + return crc32 == checksum -SYMBOLS = [ "tBTCUSD", "tLTCUSD", "tLTCBTC", "tETHUSD", "tETHBTC" ] +SYMBOLS = [ "tLTCBTC", "tETHUSD", "tETHBTC" ] raw_order_book = RawOrderBook(symbols=SYMBOLS) From 8a1632d3c21812f900348b0fa3d0c5a2325e8e06 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 1 Oct 2023 21:27:46 +0200 Subject: [PATCH 33/65] Write new implementation for class BfxEventEmitter (bfxapi.websocket._event_emitter). --- .pylintrc | 2 +- README.md | 5 -- .../websocket/_client/bfx_websocket_client.py | 62 ++++----------- .../_event_emitter/bfx_event_emitter.py | 78 +++++++++++++++---- bfxapi/websocket/_handlers/__init__.py | 1 + .../_handlers/auth_events_handler.py | 34 ++------ .../_handlers/public_channels_handler.py | 17 ---- bfxapi/websocket/exceptions.py | 4 +- 8 files changed, 87 insertions(+), 116 deletions(-) diff --git a/.pylintrc b/.pylintrc index 996e616..1f1a19b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -24,7 +24,7 @@ max-line-length=120 expected-line-ending-format=LF [BASIC] -good-names=id,on,pl,t,ip,tf,A,B,C,D,E,F +good-names=t,f,id,ip,on,pl,tf,A,B,C,D,E,F [TYPECHECK] generated-members=websockets diff --git a/README.md b/README.md index 4c181ce..68500c0 100644 --- a/README.md +++ b/README.md @@ -242,11 +242,6 @@ The same can be done without using decorators: bfx.wss.on("candles_update", callback=on_candles_update) ``` -You can pass any number of events to register for the same callback function: -```python -bfx.wss.on("t_ticker_update", "f_ticker_update", callback=on_ticker_update) -``` - # Advanced features ## Using custom notifications diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 178268c..cd95c76 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -1,7 +1,7 @@ from typing import \ - TYPE_CHECKING, TypeVar, TypedDict,\ - Callable, Optional, Tuple, \ - List, Dict, Any + TYPE_CHECKING, TypedDict, List, \ + Dict, Optional, Any, \ + no_type_check from logging import Logger from datetime import datetime @@ -20,19 +20,16 @@ from websockets.legacy.client import \ connect as _websockets__connect from bfxapi._utils.json_encoder import JSONEncoder -from bfxapi.websocket._connection import Connection -from bfxapi.websocket._event_emitter import BfxEventEmitter -from bfxapi.websocket._handlers import \ - PublicChannelsHandler, \ - AuthEventsHandler +from bfxapi.websocket._connection import Connection +from bfxapi.websocket._handlers import AuthEventsHandler +from bfxapi.websocket._event_emitter import BfxEventEmitter from bfxapi.websocket.exceptions import \ InvalidAuthenticationCredentials, \ ReconnectionTimeoutError, \ OutdatedClientVersion, \ - ZeroConnectionsError, \ - EventNotSupported + ZeroConnectionsError from .bfx_websocket_bucket import BfxWebSocketBucket @@ -43,8 +40,6 @@ if TYPE_CHECKING: from asyncio import Task - _T = TypeVar("_T", bound=Callable[..., None]) - _Reconnection = TypedDict("_Reconnection", { "attempts": int, "reason": str, "timestamp": datetime }) @@ -55,18 +50,6 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): MAXIMUM_CONNECTIONS_AMOUNT = 20 - __ONCE_EVENTS = [ - "open", "authenticated", "disconnection", - *AuthEventsHandler.ONCE_EVENTS - ] - - EVENTS = [ - "subscribed", "wss-error", - *__ONCE_EVENTS, - *PublicChannelsHandler.EVENTS, - *AuthEventsHandler.ON_EVENTS - ] - def __init__(self, host: str, *, @@ -82,9 +65,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): self.__reconnection: Optional[_Reconnection] = None - self.__event_emitter = BfxEventEmitter(targets = \ - PublicChannelsHandler.ONCE_PER_SUBSCRIPTION + \ - ["subscribed"]) + self.__event_emitter = BfxEventEmitter(loop=None) self.__handler = AuthEventsHandler( \ event_emitter=self.__event_emitter) @@ -92,7 +73,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): self.__inputs = BfxWebSocketInputs( \ handle_websocket_input=self.__handle_websocket_input) - @self.__event_emitter.on("error") + @self.__event_emitter.listens_to("error") def error(exception: Exception) -> None: header = f"{type(exception).__name__}: {str(exception)}" @@ -123,7 +104,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): _bucket = BfxWebSocketBucket( \ self._host, self.__event_emitter) - self.__buckets.update( { _bucket: None }) + self.__buckets.update({ _bucket: None }) await self.__connect() @@ -340,26 +321,9 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): await self._websocket.send(json.dumps(\ [ 0, event, None, data], cls=JSONEncoder)) - def on(self, *events: str, callback: Optional["_T"] = None) -> Callable[["_T"], None]: - for event in events: - if event not in BfxWebSocketClient.EVENTS: - raise EventNotSupported(f"Event <{event}> is not supported. To get a list " \ - "of available events see BfxWebSocketClient.EVENTS.") - - def _register_events(function: "_T", events: Tuple[str, ...]) -> None: - for event in events: - if event in BfxWebSocketClient.__ONCE_EVENTS: - self.__event_emitter.once(event, function) - else: - self.__event_emitter.on(event, function) - - if callback: - _register_events(callback, events) - - def _handler(function: "_T") -> None: - _register_events(function, events) - - return _handler + @no_type_check + def on(self, event, f = None): + return self.__event_emitter.on(event, f=f) @staticmethod def __build_authentication_message(api_key: str, diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py index 9ebabd2..ec594ab 100644 --- a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -1,37 +1,83 @@ from typing import \ - TYPE_CHECKING, List, Dict, Any + Callable, List, Dict, \ + Optional, Any from collections import defaultdict - +from asyncio import AbstractEventLoop from pyee.asyncio import AsyncIOEventEmitter -if TYPE_CHECKING: - from bfxapi.websocket.subscriptions import Subscription +from bfxapi.websocket.exceptions import UnknownEventError + +_ONCE_PER_CONNECTION = [ + "open", "authenticated", "disconnection", + "order_snapshot", "position_snapshot", "funding_offer_snapshot", + "funding_credit_snapshot", "funding_loan_snapshot", "wallet_snapshot" +] + +_ONCE_PER_SUBSCRIPTION = [ + "subscribed", "t_trades_snapshot", "f_trades_snapshot", + "t_book_snapshot", "f_book_snapshot", "t_raw_book_snapshot", + "f_raw_book_snapshot", "candles_snapshot" +] + +_COMMON = [ + "error", "wss-error", "t_ticker_update", + "f_ticker_update", "t_trade_execution", "t_trade_execution_update", + "f_trade_execution", "f_trade_execution_update", "t_book_update", + "f_book_update", "t_raw_book_update", "f_raw_book_update", + "candles_update", "derivatives_status_update", "liquidation_feed_update", + "order_new", "order_update", "order_cancel", + "position_new", "position_update", "position_close", + "funding_offer_new", "funding_offer_update", "funding_offer_cancel", + "funding_credit_new", "funding_credit_update", "funding_credit_close", + "funding_loan_new", "funding_loan_update", "funding_loan_close", + "trade_execution", "trade_execution_update", "wallet_update", + "notification", "on-req-notification", "ou-req-notification", + "oc-req-notification", "fon-req-notification", "foc-req-notification" +] class BfxEventEmitter(AsyncIOEventEmitter): - def __init__(self, targets: List[str]) -> None: - super().__init__() + _EVENTS = _ONCE_PER_CONNECTION + \ + _ONCE_PER_SUBSCRIPTION + \ + _COMMON - self.__targets = targets + def __init__(self, loop: Optional[AbstractEventLoop] = None) -> None: + super().__init__(loop) - self.__log: Dict[str, List[str]] = \ + self._connection: List[str] = [ ] + + self._subscriptions: Dict[str, List[str]] = \ defaultdict(lambda: [ ]) def emit(self, event: str, *args: Any, **kwargs: Any) -> bool: - if event in self.__targets: - subscription: "Subscription" = args[0] + if event in _ONCE_PER_CONNECTION: + if event in self._connection: + return self._has_listeners(event) - sub_id = subscription["subId"] + self._connection += [ event ] - if event in self.__log[sub_id]: - with self._lock: - listeners = self._events.get(event) + if event in _ONCE_PER_SUBSCRIPTION: + sub_id = args[0]["subId"] - return bool(listeners) + if event in self._subscriptions[sub_id]: + return self._has_listeners(event) - self.__log[sub_id] += [ event ] + self._subscriptions[sub_id] += [ event ] return super().emit(event, *args, **kwargs) + + def _add_event_handler(self, event: str, k: Callable, v: Callable): + if event not in BfxEventEmitter._EVENTS: + raise UnknownEventError(f"Can't register to unknown event: <{event}> " + \ + "(to get a full list of available events see https://docs.bitfinex.com/).") + + super()._add_event_handler(event, k, v) + + def _has_listeners(self, event: str) -> bool: + with self._lock: + listeners = self._events.get(event) + + return bool(listeners) diff --git a/bfxapi/websocket/_handlers/__init__.py b/bfxapi/websocket/_handlers/__init__.py index 23f5aad..12a1dd1 100644 --- a/bfxapi/websocket/_handlers/__init__.py +++ b/bfxapi/websocket/_handlers/__init__.py @@ -1,2 +1,3 @@ from .public_channels_handler import PublicChannelsHandler + from .auth_events_handler import AuthEventsHandler diff --git a/bfxapi/websocket/_handlers/auth_events_handler.py b/bfxapi/websocket/_handlers/auth_events_handler.py index 7028043..18940f2 100644 --- a/bfxapi/websocket/_handlers/auth_events_handler.py +++ b/bfxapi/websocket/_handlers/auth_events_handler.py @@ -12,35 +12,17 @@ if TYPE_CHECKING: from pyee.base import EventEmitter class AuthEventsHandler: - __ONCE_ABBREVIATIONS = { - "os": "order_snapshot", "ps": "position_snapshot", "fos": "funding_offer_snapshot", - "fcs": "funding_credit_snapshot", "fls": "funding_loan_snapshot", "ws": "wallet_snapshot" - } - - __ON_ABBREVIATIONS = { - "on": "order_new", "ou": "order_update", "oc": "order_cancel", - "pn": "position_new", "pu": "position_update", "pc": "position_close", - "fon": "funding_offer_new", "fou": "funding_offer_update", "foc": "funding_offer_cancel", - "fcn": "funding_credit_new", "fcu": "funding_credit_update", "fcc": "funding_credit_close", - "fln": "funding_loan_new", "flu": "funding_loan_update", "flc": "funding_loan_close", - "te": "trade_execution", "tu": "trade_execution_update", "wu": "wallet_update" - } - __ABBREVIATIONS = { - **__ONCE_ABBREVIATIONS, - **__ON_ABBREVIATIONS + "os": "order_snapshot", "on": "order_new", "ou": "order_update", + "oc": "order_cancel", "ps": "position_snapshot", "pn": "position_new", + "pu": "position_update", "pc": "position_close", "te": "trade_execution", + "tu": "trade_execution_update", "fos": "funding_offer_snapshot", "fon": "funding_offer_new", + "fou": "funding_offer_update", "foc": "funding_offer_cancel", "fcs": "funding_credit_snapshot", + "fcn": "funding_credit_new", "fcu": "funding_credit_update", "fcc": "funding_credit_close", + "fls": "funding_loan_snapshot", "fln": "funding_loan_new", "flu": "funding_loan_update", + "flc": "funding_loan_close", "ws": "wallet_snapshot", "wu": "wallet_update" } - ONCE_EVENTS = [ - *list(__ONCE_ABBREVIATIONS.values()) - ] - - ON_EVENTS = [ - *list(__ON_ABBREVIATIONS.values()), - "notification", "on-req-notification", "ou-req-notification", - "oc-req-notification", "fon-req-notification", "foc-req-notification" - ] - def __init__(self, event_emitter: "EventEmitter") -> None: self.__event_emitter = event_emitter diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index 0bcd805..5bc22f2 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -15,23 +15,6 @@ if TYPE_CHECKING: _CHECKSUM = "cs" class PublicChannelsHandler: - ONCE_PER_SUBSCRIPTION = [ - "t_trades_snapshot", "f_trades_snapshot", "t_book_snapshot", - "f_book_snapshot", "t_raw_book_snapshot", "f_raw_book_snapshot", - "candles_snapshot" - ] - - EVENTS = [ - *ONCE_PER_SUBSCRIPTION, - "t_ticker_update", "f_ticker_update", "t_trade_execution", - "t_trade_execution_update", "f_trade_execution", "f_trade_execution_update", - "t_book_update", "f_book_update", "t_raw_book_update", - "f_raw_book_update", "candles_update", "derivatives_status_update", - "liquidation_feed_update", - - "checksum" - ] - def __init__(self, event_emitter: "EventEmitter") -> None: self.__event_emitter = event_emitter diff --git a/bfxapi/websocket/exceptions.py b/bfxapi/websocket/exceptions.py index 6ed6f3f..6469723 100644 --- a/bfxapi/websocket/exceptions.py +++ b/bfxapi/websocket/exceptions.py @@ -9,7 +9,7 @@ __all__ = [ "ReconnectionTimeoutError", "ActionRequiresAuthentication", "InvalidAuthenticationCredentials", - "EventNotSupported", + "UnknownEventError", "OutdatedClientVersion" ] @@ -48,7 +48,7 @@ class InvalidAuthenticationCredentials(BfxWebSocketException): This error indicates that the user has provided incorrect credentials (API-KEY and API-SECRET) for authentication. """ -class EventNotSupported(BfxWebSocketException): +class UnknownEventError(BfxWebSocketException): """ This error indicates a failed attempt to subscribe to an event not supported by the BfxWebSocketClient. """ From ca4050a35b2eccd5f85e2a4afa084e9a9b3a2a0a Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 1 Oct 2023 21:37:43 +0200 Subject: [PATCH 34/65] Rename event to (to mantain compliance). --- README.md | 6 +++--- bfxapi/websocket/_client/bfx_websocket_client.py | 2 +- bfxapi/websocket/_event_emitter/bfx_event_emitter.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 68500c0..3116e38 100644 --- a/README.md +++ b/README.md @@ -181,10 +181,10 @@ A custom [close code number](https://www.iana.org/assignments/websocket/websocke await bfx.wss.close(code=1001, reason="Going Away") ``` -After closing the connection, the client will emit the `disconnection` event: +After closing the connection, the client will emit the `disconnected` event: ```python -@bfx.wss.on("disconnection") -def on_disconnection(code: int, reason: str): +@bfx.wss.on("disconnected") +def on_disconnected(code: int, reason: str): if code == 1000 or code == 1001: print("Closing the connection without errors!") ``` diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index cd95c76..61431b5 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -206,7 +206,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): raise error if not self.__reconnection: - self.__event_emitter.emit("disconnection", + self.__event_emitter.emit("disconnected", self._websocket.close_code, \ self._websocket.close_reason) diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py index ec594ab..3c11c08 100644 --- a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -9,7 +9,7 @@ from pyee.asyncio import AsyncIOEventEmitter from bfxapi.websocket.exceptions import UnknownEventError _ONCE_PER_CONNECTION = [ - "open", "authenticated", "disconnection", + "open", "authenticated", "disconnected", "order_snapshot", "position_snapshot", "funding_offer_snapshot", "funding_credit_snapshot", "funding_loan_snapshot", "wallet_snapshot" ] From 82a3307205aa2601445497373fe651e55439b5e8 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 1 Oct 2023 22:33:08 +0200 Subject: [PATCH 35/65] Fix bug in local class _Delay (bfx_websocket_client.py). --- .../websocket/_client/bfx_websocket_client.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 61431b5..3b9594d 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -45,6 +45,30 @@ if TYPE_CHECKING: _DEFAULT_LOGGER = Logger("bfxapi.websocket._client", level=0) +class _Delay: + __BACKOFF_MIN = 1.92 + + __BACKOFF_MAX = 60.0 + + def __init__(self, backoff_factor: float) -> None: + self.__backoff_factor = backoff_factor + self.__backoff_delay = _Delay.__BACKOFF_MIN + self.__initial_delay = random.uniform(1.0, 5.0) + + def next(self) -> float: + _backoff_delay = self.peek() + __backoff_delay = _backoff_delay * self.__backoff_factor + self.__backoff_delay = min(__backoff_delay, _Delay.__BACKOFF_MAX) + + return _backoff_delay + + def peek(self) -> float: + return (self.__backoff_delay == _Delay.__BACKOFF_MIN) \ + and self.__initial_delay or self.__backoff_delay + + def reset(self) -> None: + self.__backoff_delay = _Delay.__BACKOFF_MIN + class BfxWebSocketClient(Connection, Connection.Authenticable): VERSION = BfxWebSocketBucket.VERSION @@ -108,32 +132,8 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): await self.__connect() - #pylint: disable-next=too-many-branches,too-many-statements + #pylint: disable-next=too-many-branches async def __connect(self) -> None: - class _Delay: - BACKOFF_MIN, BACKOFF_MAX = 1.92, 60.0 - - BACKOFF_INITIAL = 5.0 - - def __init__(self, backoff_factor: float) -> None: - self.__backoff_factor = backoff_factor - self.__backoff_delay = _Delay.BACKOFF_MIN - self.__initial_delay = random.random() * _Delay.BACKOFF_INITIAL - - def next(self) -> float: - _backoff_delay = self.peek() - __backoff_delay = self.__backoff_delay * self.__backoff_factor - self.__backoff_delay = min(__backoff_delay, _Delay.BACKOFF_MAX) - - return _backoff_delay - - def peek(self) -> float: - return (self.__backoff_delay == _Delay.BACKOFF_MIN) \ - and self.__initial_delay or self.__backoff_delay - - def reset(self) -> None: - self.__backoff_delay = _Delay.BACKOFF_MIN - _delay = _Delay(backoff_factor=1.618) _sleep: Optional["Task"] = None @@ -146,7 +146,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): while True: if self.__reconnection: _sleep = asyncio.create_task( \ - asyncio.sleep(_delay.next())) + asyncio.sleep(int(_delay.next()))) try: await _sleep @@ -198,7 +198,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): isinstance(error, gaierror)) and self.__reconnection: self.__logger.warning( f"_Reconnection attempt was unsuccessful (no.{self.__reconnection['attempts']}). " \ - f"Next reconnection attempt in {_delay.peek():.2f} seconds. (at the moment " \ + f"Next reconnection attempt in {int(_delay.peek())}.0 seconds. (at the moment " \ f"the client has been offline for {datetime.now() - self.__reconnection['timestamp']})") self.__reconnection["attempts"] += 1 From 22451f674ed8611bd54b6ea04d5313df324f4022 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 1 Oct 2023 22:36:47 +0200 Subject: [PATCH 36/65] Remove inner class Connection.Authenticable (_connection.py). --- .../websocket/_client/bfx_websocket_client.py | 6 +-- bfxapi/websocket/_connection.py | 40 +++++++++---------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 3b9594d..8b6aada 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -69,7 +69,7 @@ class _Delay: def reset(self) -> None: self.__backoff_delay = _Delay.__BACKOFF_MIN -class BfxWebSocketClient(Connection, Connection.Authenticable): +class BfxWebSocketClient(Connection): VERSION = BfxWebSocketBucket.VERSION MAXIMUM_CONNECTIONS_AMOUNT = 20 @@ -307,7 +307,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): await self._websocket.close( \ code=code, reason=reason) - @Connection.Authenticable.require_websocket_authentication + @Connection.require_websocket_authentication async def notify(self, info: Any, message_id: Optional[int] = None, @@ -316,7 +316,7 @@ class BfxWebSocketClient(Connection, Connection.Authenticable): json.dumps([ 0, "n", message_id, { "type": "ucm-test", "info": info, **kwargs } ])) - @Connection.Authenticable.require_websocket_authentication + @Connection.require_websocket_authentication async def __handle_websocket_input(self, event: str, data: Any) -> None: await self._websocket.send(json.dumps(\ [ 0, event, None, data], cls=JSONEncoder)) diff --git a/bfxapi/websocket/_connection.py b/bfxapi/websocket/_connection.py index 1562b43..f9dbd3c 100644 --- a/bfxapi/websocket/_connection.py +++ b/bfxapi/websocket/_connection.py @@ -10,30 +10,11 @@ if TYPE_CHECKING: class Connection: HEARTBEAT = "hb" - class Authenticable: - def __init__(self) -> None: - self._authentication: bool = False - - @property - def authentication(self) -> bool: - return self._authentication - - @staticmethod - def require_websocket_authentication(function): - async def wrapper(self, *args, **kwargs): - if not self.authentication: - raise ActionRequiresAuthentication("To perform this action you need to " \ - "authenticate using your API_KEY and API_SECRET.") - - internal = Connection.require_websocket_connection(function) - - return await internal(self, *args, **kwargs) - - return wrapper - def __init__(self, host: str) -> None: self._host = host + self._authentication: bool = False + self.__protocol: Optional["WebSocketClientProtocol"] = None @property @@ -41,6 +22,10 @@ class Connection: return self.__protocol is not None and \ self.__protocol.open + @property + def authentication(self) -> bool: + return self._authentication + @property def _websocket(self) -> "WebSocketClientProtocol": return cast("WebSocketClientProtocol", self.__protocol) @@ -58,3 +43,16 @@ class Connection: raise ConnectionNotOpen("No open connection with the server.") return wrapper + + @staticmethod + def require_websocket_authentication(function): + async def wrapper(self, *args, **kwargs): + if not self.authentication: + raise ActionRequiresAuthentication("To perform this action you need to " \ + "authenticate using your API_KEY and API_SECRET.") + + internal = Connection.require_websocket_connection(function) + + return await internal(self, *args, **kwargs) + + return wrapper From 206ebe79415867d25f4fde1a80bf36feb482a5f6 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 1 Oct 2023 23:01:17 +0200 Subject: [PATCH 37/65] Remove circular import from file bfx_websocket_client.py. --- bfxapi/client.py | 6 +++--- bfxapi/websocket/_client/bfx_websocket_client.py | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bfxapi/client.py b/bfxapi/client.py index dd9127c..3d7f3e6 100644 --- a/bfxapi/client.py +++ b/bfxapi/client.py @@ -1,5 +1,5 @@ from typing import \ - TYPE_CHECKING, TypedDict, List, Literal, Optional + TYPE_CHECKING, List, Literal, Optional from bfxapi._utils.logging import ColorLogger @@ -10,8 +10,8 @@ from bfxapi.websocket import BfxWebSocketClient from bfxapi.urls import REST_HOST, WSS_HOST if TYPE_CHECKING: - _Credentials = TypedDict("_Credentials", \ - { "api_key": str, "api_secret": str, "filters": Optional[List[str]] }) + from bfxapi.websocket._client.bfx_websocket_client import \ + _Credentials class Client: def __init__( diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 8b6aada..249ba59 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -31,15 +31,16 @@ from bfxapi.websocket.exceptions import \ OutdatedClientVersion, \ ZeroConnectionsError -from .bfx_websocket_bucket import BfxWebSocketBucket +from bfxapi.websocket._client.bfx_websocket_bucket import BfxWebSocketBucket -from .bfx_websocket_inputs import BfxWebSocketInputs +from bfxapi.websocket._client.bfx_websocket_inputs import BfxWebSocketInputs if TYPE_CHECKING: - from bfxapi.client import _Credentials - from asyncio import Task + _Credentials = TypedDict("_Credentials", \ + { "api_key": str, "api_secret": str, "filters": Optional[List[str]] }) + _Reconnection = TypedDict("_Reconnection", { "attempts": int, "reason": str, "timestamp": datetime }) From 628c3a0d66eb3ac30e30b13ecaf2bc351f3cd33e Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 2 Oct 2023 19:25:13 +0200 Subject: [PATCH 38/65] Rewrite implementation for abstract class Connection (_connection.py). --- bfxapi/websocket/_connection.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/bfxapi/websocket/_connection.py b/bfxapi/websocket/_connection.py index f9dbd3c..4ef6fad 100644 --- a/bfxapi/websocket/_connection.py +++ b/bfxapi/websocket/_connection.py @@ -1,5 +1,11 @@ from typing import \ - TYPE_CHECKING, Optional, cast + TYPE_CHECKING, TypeVar, Callable, \ + Awaitable, Optional, Any, \ + cast + +from abc import ABC, abstractmethod + +from typing_extensions import ParamSpec, Concatenate from bfxapi.websocket.exceptions import \ ConnectionNotOpen, ActionRequiresAuthentication @@ -7,7 +13,13 @@ from bfxapi.websocket.exceptions import \ if TYPE_CHECKING: from websockets.client import WebSocketClientProtocol -class Connection: +_S = TypeVar("_S", bound="Connection") + +_R = TypeVar("_R") + +_P = ParamSpec("_P") + +class Connection(ABC): HEARTBEAT = "hb" def __init__(self, host: str) -> None: @@ -34,9 +46,15 @@ class Connection: def _websocket(self, protocol: "WebSocketClientProtocol") -> None: self.__protocol = protocol + @abstractmethod + async def start(self) -> None: + ... + @staticmethod - def require_websocket_connection(function): - async def wrapper(self, *args, **kwargs): + def require_websocket_connection( + function: Callable[Concatenate[_S, _P], Awaitable[_R]] + ) -> Callable[Concatenate[_S, _P], Awaitable["_R"]]: + async def wrapper(self: _S, *args: Any, **kwargs: Any) -> _R: if self.open: return await function(self, *args, **kwargs) @@ -45,8 +63,10 @@ class Connection: return wrapper @staticmethod - def require_websocket_authentication(function): - async def wrapper(self, *args, **kwargs): + def require_websocket_authentication( + function: Callable[Concatenate[_S, _P], Awaitable[_R]] + ) -> Callable[Concatenate[_S, _P], Awaitable[_R]]: + async def wrapper(self: _S, *args: Any, **kwargs: Any) -> _R: if not self.authentication: raise ActionRequiresAuthentication("To perform this action you need to " \ "authenticate using your API_KEY and API_SECRET.") From 5ae576e36ae98f3a6745272cfb2a87dcfdb20936 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 8 Oct 2023 05:37:10 +0200 Subject: [PATCH 39/65] Fix and rewrite all logic in class BfxWebSocketBucket. --- README.md | 2 +- bfxapi/websocket/__init__.py | 2 +- bfxapi/websocket/_client/__init__.py | 2 - .../websocket/_client/bfx_websocket_bucket.py | 135 ++++++++++-------- .../websocket/_client/bfx_websocket_client.py | 16 ++- bfxapi/websocket/_connection.py | 2 +- .../_event_emitter/bfx_event_emitter.py | 2 +- .../_handlers/public_channels_handler.py | 32 ++--- bfxapi/websocket/exceptions.py | 14 +- bfxapi/websocket/subscriptions.py | 64 +++------ examples/websocket/public/order_book.py | 2 +- examples/websocket/public/raw_order_book.py | 2 +- examples/websocket/public/ticker.py | 2 +- 13 files changed, 132 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index 3116e38..004b346 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ On each successful subscription, the client will emit the `subscribed` event: @bfx.wss.on("subscribed") def on_subscribed(subscription: subscriptions.Subscription): if subscription["channel"] == "ticker": - print(f"{subscription['symbol']}: {subscription['subId']}") # tBTCUSD: f2757df2-7e11-4244-9bb7-a53b7343bef8 + print(f"{subscription['symbol']}: {subscription['sub_id']}") # tBTCUSD: f2757df2-7e11-4244-9bb7-a53b7343bef8 ``` ### Unsubscribing from a public channel diff --git a/bfxapi/websocket/__init__.py b/bfxapi/websocket/__init__.py index f1ed659..ced8300 100644 --- a/bfxapi/websocket/__init__.py +++ b/bfxapi/websocket/__init__.py @@ -1 +1 @@ -from ._client import BfxWebSocketClient, BfxWebSocketBucket, BfxWebSocketInputs +from ._client import BfxWebSocketClient diff --git a/bfxapi/websocket/_client/__init__.py b/bfxapi/websocket/_client/__init__.py index 05b843c..ebbd6d2 100644 --- a/bfxapi/websocket/_client/__init__.py +++ b/bfxapi/websocket/_client/__init__.py @@ -1,3 +1 @@ from .bfx_websocket_client import BfxWebSocketClient -from .bfx_websocket_bucket import BfxWebSocketBucket -from .bfx_websocket_inputs import BfxWebSocketInputs diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 0130dfb..3c42fdf 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -1,32 +1,32 @@ from typing import \ - TYPE_CHECKING, Optional, Dict, List, Any, cast + List, Dict, Any, \ + Optional, cast import asyncio, json, uuid -from websockets.legacy.client import connect as _websockets__connect +import websockets.client +from pyee import EventEmitter from bfxapi.websocket._connection import Connection from bfxapi.websocket._handlers import PublicChannelsHandler -from bfxapi.websocket.exceptions import TooManySubscriptions +from bfxapi.websocket.subscriptions import Subscription -if TYPE_CHECKING: - from bfxapi.websocket.subscriptions import Subscription - from websockets.client import WebSocketClientProtocol - from pyee import EventEmitter +from bfxapi.websocket.exceptions import FullBucketError _CHECKSUM_FLAG_VALUE = 131_072 +def _strip(message: Dict[str, Any], keys: List[str]) -> Dict[str, Any]: + return { key: message[key] for key in message if not key in keys } + class BfxWebSocketBucket(Connection): - VERSION = 2 + __MAXIMUM_SUBSCRIPTIONS_AMOUNT = 25 - MAXIMUM_SUBSCRIPTIONS_AMOUNT = 25 - - def __init__(self, host: str, event_emitter: "EventEmitter") -> None: + def __init__(self, host: str, event_emitter: EventEmitter) -> None: super().__init__(host) self.__event_emitter = event_emitter self.__pendings: List[Dict[str, Any]] = [ ] - self.__subscriptions: Dict[int, "Subscription"] = { } + self.__subscriptions: Dict[int, Subscription] = { } self.__condition = asyncio.locks.Condition() @@ -34,118 +34,131 @@ class BfxWebSocketBucket(Connection): event_emitter=self.__event_emitter) @property - def pendings(self) -> List[Dict[str, Any]]: - return self.__pendings + def count(self) -> int: + return len(self.__pendings) + \ + len(self.__subscriptions) @property - def subscriptions(self) -> Dict[int, "Subscription"]: - return self.__subscriptions + def is_full(self) -> bool: + return self.count == \ + BfxWebSocketBucket.__MAXIMUM_SUBSCRIPTIONS_AMOUNT - async def connect(self) -> None: - async with _websockets__connect(self._host) as websocket: + async def start(self) -> None: + async with websockets.client.connect(self._host) as websocket: self._websocket = websocket await self.__recover_state() - await self.__set_conf(flags=_CHECKSUM_FLAG_VALUE) - async with self.__condition: self.__condition.notify(1) - async for message in self._websocket: - message = json.loads(message) + async for _message in self._websocket: + message = json.loads(_message) if isinstance(message, dict): - if message["event"] == "subscribed" and (chan_id := message["chanId"]): - self.__pendings = [ pending \ - for pending in self.__pendings \ - if pending["subId"] != message["subId"] ] + # I think there's a better way to do it... + if "chanId" in message: + message["chan_id"] = message.pop("chanId") - self.__subscriptions[chan_id] = cast("Subscription", message) + if "subId" in message: + message["sub_id"] = message.pop("subId") - self.__event_emitter.emit("subscribed", message) - elif message["event"] == "unsubscribed" and (chan_id := message["chanId"]): + if message["event"] == "subscribed": + self.__on_subscribed(message) + elif message["event"] == "unsubscribed": if message["status"] == "OK": + chan_id = cast(int, message["chan_id"]) + del self.__subscriptions[chan_id] elif message["event"] == "error": - self.__event_emitter.emit( \ - "wss-error", message["code"], message["msg"]) + self.__event_emitter.emit("wss-error", \ + message["code"], message["msg"]) if isinstance(message, list): - if (chan_id := message[0]) and message[1] != Connection.HEARTBEAT: + if (chan_id := cast(int, message[0])) and \ + (message[1] != Connection._HEARTBEAT): self.__handler.handle(self.__subscriptions[chan_id], message[1:]) + def __on_subscribed(self, message: Dict[str, Any]) -> None: + chan_id = cast(int, message["chan_id"]) + + subscription = cast(Subscription, _strip(message, \ + keys=["event", "chan_id", "pair", "currency"])) + + self.__pendings = [ pending \ + for pending in self.__pendings \ + if pending["subId"] != message["sub_id"] ] + + self.__subscriptions[chan_id] = subscription + + self.__event_emitter.emit("subscribed", subscription) + async def __recover_state(self) -> None: for pending in self.__pendings: - await self._websocket.send( \ + await self._websocket.send(message = \ json.dumps(pending)) - for _, subscription in self.__subscriptions.items(): - _subscription = cast(Dict[str, Any], subscription) + for chan_id in list(self.__subscriptions.keys()): + subscription = self.__subscriptions.pop(chan_id) - await self.subscribe( \ - sub_id=_subscription.pop("subId"), - **_subscription) + await self.subscribe(**subscription) - self.__subscriptions.clear() + await self.__set_config([ _CHECKSUM_FLAG_VALUE ]) - async def __set_conf(self, flags: int) -> None: + async def __set_config(self, flags: List[int]) -> None: await self._websocket.send(json.dumps( \ - { "event": "conf", "flags": flags })) + { "event": "conf", "flags": sum(flags) })) @Connection.require_websocket_connection async def subscribe(self, channel: str, sub_id: Optional[str] = None, **kwargs: Any) -> None: - if len(self.__subscriptions) + len(self.__pendings) \ - == BfxWebSocketBucket.MAXIMUM_SUBSCRIPTIONS_AMOUNT: - raise TooManySubscriptions("The client has reached the maximum number of subscriptions.") + if self.is_full: + raise FullBucketError("The bucket is full: " + \ + "can't subscribe to any other channel.") - subscription = \ + subscription: Dict[str, Any] = \ { **kwargs, "event": "subscribe", "channel": channel } subscription["subId"] = sub_id or str(uuid.uuid4()) self.__pendings.append(subscription) - await self._websocket.send( \ + await self._websocket.send(message = \ json.dumps(subscription)) @Connection.require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: for chan_id, subscription in self.__subscriptions.items(): - if subscription["subId"] == sub_id: - message = json.dumps({ + if subscription["sub_id"] == sub_id: + unsubscription = { "event": "unsubscribe", - "chanId": chan_id }) + "chanId": chan_id } - await self._websocket.send(message) + await self._websocket.send(message = \ + json.dumps(unsubscription)) @Connection.require_websocket_connection async def resubscribe(self, sub_id: str) -> None: for subscription in self.__subscriptions.values(): - if subscription["subId"] == sub_id: - _subscription = cast(Dict[str, Any], subscription) + if subscription["sub_id"] == sub_id: + await self.unsubscribe(sub_id) - await self.unsubscribe(sub_id=sub_id) - - await self.subscribe( \ - sub_id=_subscription.pop("subId"), - **_subscription) + await self.subscribe(**subscription) @Connection.require_websocket_connection async def close(self, code: int = 1000, reason: str = str()) -> None: - await self._websocket.close(code=code, reason=reason) + await self._websocket.close(code, reason) def has(self, sub_id: str) -> bool: for subscription in self.__subscriptions.values(): - if subscription["subId"] == sub_id: + if subscription["sub_id"] == sub_id: return True return False async def wait(self) -> None: async with self.__condition: - await self.__condition.wait_for( - lambda: self.open) + await self.__condition \ + .wait_for(lambda: self.open) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 249ba59..1518e35 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -29,7 +29,8 @@ from bfxapi.websocket.exceptions import \ InvalidAuthenticationCredentials, \ ReconnectionTimeoutError, \ OutdatedClientVersion, \ - ZeroConnectionsError + ZeroConnectionsError, \ + UnknownChannelError from bfxapi.websocket._client.bfx_websocket_bucket import BfxWebSocketBucket @@ -71,7 +72,7 @@ class _Delay: self.__backoff_delay = _Delay.__BACKOFF_MIN class BfxWebSocketClient(Connection): - VERSION = BfxWebSocketBucket.VERSION + VERSION = 2 MAXIMUM_CONNECTIONS_AMOUNT = 20 @@ -227,7 +228,7 @@ class BfxWebSocketClient(Connection): self.__buckets = { bucket: asyncio.create_task(_c) for bucket in self.__buckets - if (_c := bucket.connect()) + if (_c := bucket.start()) } if len(self.__buckets) == 0 or \ @@ -265,7 +266,7 @@ class BfxWebSocketClient(Connection): self.__event_emitter.emit("wss-error", message["code"], message["msg"]) if isinstance(message, list) and \ - message[0] == 0 and message[1] != Connection.HEARTBEAT: + message[0] == 0 and message[1] != Connection._HEARTBEAT: self.__handler.handle(message[1], message[2]) @Connection.require_websocket_connection @@ -277,10 +278,13 @@ class BfxWebSocketClient(Connection): raise ZeroConnectionsError("Unable to subscribe: " \ "the number of connections must be greater than 0.") + if not channel in ["ticker", "trades", "book", "candles", "status"]: + raise UnknownChannelError("Available channels are: " + \ + "ticker, trades, book, candles and status.") + _buckets = list(self.__buckets.keys()) - counters = [ len(bucket.pendings) + len(bucket.subscriptions) - for bucket in _buckets ] + counters = [ bucket.count for bucket in _buckets ] index = counters.index(min(counters)) diff --git a/bfxapi/websocket/_connection.py b/bfxapi/websocket/_connection.py index 4ef6fad..971bdd0 100644 --- a/bfxapi/websocket/_connection.py +++ b/bfxapi/websocket/_connection.py @@ -20,7 +20,7 @@ _R = TypeVar("_R") _P = ParamSpec("_P") class Connection(ABC): - HEARTBEAT = "hb" + _HEARTBEAT = "hb" def __init__(self, host: str) -> None: self._host = host diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py index 3c11c08..e53eee8 100644 --- a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -60,7 +60,7 @@ class BfxEventEmitter(AsyncIOEventEmitter): self._connection += [ event ] if event in _ONCE_PER_SUBSCRIPTION: - sub_id = args[0]["subId"] + sub_id = args[0]["sub_id"] if event in self._subscriptions[sub_id]: return self._has_listeners(event) diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index 5bc22f2..597f16c 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -1,5 +1,6 @@ -from typing import TYPE_CHECKING, \ - Union, List, Any, cast +from typing import \ + TYPE_CHECKING, List, Any, \ + cast from bfxapi.types import serializers @@ -9,9 +10,6 @@ if TYPE_CHECKING: from pyee.base import EventEmitter - _NoHeaderSubscription = \ - Union[Ticker, Trades, Book, Candles, Status] - _CHECKSUM = "cs" class PublicChannelsHandler: @@ -19,30 +17,24 @@ class PublicChannelsHandler: self.__event_emitter = event_emitter def handle(self, subscription: "Subscription", stream: List[Any]) -> None: - def _strip(subscription: "Subscription", *args: str) -> "_NoHeaderSubscription": - return cast("_NoHeaderSubscription", \ - { key: value for key, value in subscription.items() if key not in args }) - - _subscription = _strip(subscription, "event", "channel", "chanId") - if subscription["channel"] == "ticker": - self.__ticker_channel_handler(cast("Ticker", _subscription), stream) + self.__ticker_channel_handler(cast("Ticker", subscription), stream) elif subscription["channel"] == "trades": - self.__trades_channel_handler(cast("Trades", _subscription), stream) + self.__trades_channel_handler(cast("Trades", subscription), stream) elif subscription["channel"] == "book": - _subscription = cast("Book", _subscription) + subscription = cast("Book", subscription) if stream[0] == _CHECKSUM: - self.__checksum_handler(_subscription, stream[1]) + self.__checksum_handler(subscription, stream[1]) else: - if _subscription["prec"] != "R0": - self.__book_channel_handler(_subscription, stream) + if subscription["prec"] != "R0": + self.__book_channel_handler(subscription, stream) else: - self.__raw_book_channel_handler(_subscription, stream) + self.__raw_book_channel_handler(subscription, stream) elif subscription["channel"] == "candles": - self.__candles_channel_handler(cast("Candles", _subscription), stream) + self.__candles_channel_handler(cast("Candles", subscription), stream) elif subscription["channel"] == "status": - self.__status_channel_handler(cast("Status", _subscription), stream) + self.__status_channel_handler(cast("Status", subscription), stream) def __ticker_channel_handler(self, subscription: "Ticker", stream: List[Any]): if subscription["symbol"].startswith("t"): diff --git a/bfxapi/websocket/exceptions.py b/bfxapi/websocket/exceptions.py index 6469723..c23da60 100644 --- a/bfxapi/websocket/exceptions.py +++ b/bfxapi/websocket/exceptions.py @@ -4,11 +4,12 @@ __all__ = [ "BfxWebSocketException", "ConnectionNotOpen", - "TooManySubscriptions", + "FullBucketError", "ZeroConnectionsError", "ReconnectionTimeoutError", "ActionRequiresAuthentication", "InvalidAuthenticationCredentials", + "UnknownChannelError", "UnknownEventError", "OutdatedClientVersion" ] @@ -23,9 +24,9 @@ class ConnectionNotOpen(BfxWebSocketException): This error indicates an attempt to communicate via websocket before starting the connection with the servers. """ -class TooManySubscriptions(BfxWebSocketException): +class FullBucketError(BfxWebSocketException): """ - This error indicates a subscription attempt after reaching the limit of simultaneous connections. + Thrown when a user attempts a subscription but all buckets are full. """ class ZeroConnectionsError(BfxWebSocketException): @@ -48,9 +49,14 @@ class InvalidAuthenticationCredentials(BfxWebSocketException): This error indicates that the user has provided incorrect credentials (API-KEY and API-SECRET) for authentication. """ +class UnknownChannelError(BfxWebSocketException): + """ + Thrown when a user attempts to subscribe to an unknown channel. + """ + class UnknownEventError(BfxWebSocketException): """ - This error indicates a failed attempt to subscribe to an event not supported by the BfxWebSocketClient. + Thrown when a user attempts to add a listener for an unknown event. """ class OutdatedClientVersion(BfxWebSocketException): diff --git a/bfxapi/websocket/subscriptions.py b/bfxapi/websocket/subscriptions.py index d4ffd2a..db3a2c9 100644 --- a/bfxapi/websocket/subscriptions.py +++ b/bfxapi/websocket/subscriptions.py @@ -1,60 +1,34 @@ -from typing import TypedDict, \ - Union, Literal, Optional +from typing import \ + Union, Literal, TypedDict + +Subscription = Union["Ticker", "Trades", "Book", "Candles", "Status"] + +Channel = Literal["ticker", "trades", "book", "candles", "status"] class Ticker(TypedDict): - subId: str + channel: Literal["ticker"] + sub_id: str symbol: str - pair: Optional[str] - currency: Optional[str] class Trades(TypedDict): - subId: str + channel: Literal["trades"] + sub_id: str symbol: str - pair: Optional[str] - currency: Optional[str] class Book(TypedDict): - subId: str + channel: Literal["book"] + sub_id: str symbol: str - prec: str - freq: str - len: str - pair: str + prec: Literal["R0", "P0", "P1", "P2", "P3", "P4"] + freq: Literal["F0", "F1"] + len: Literal["1", "25", "100", "250"] class Candles(TypedDict): - subId: str + channel: Literal["candles"] + sub_id: str key: str class Status(TypedDict): - subId: str + channel: Literal["status"] + sub_id: str key: str - -Subscription = Union["_Ticker", "_Trades", "_Book", "_Candles", "_Status"] - -_Channel = Literal["ticker", "trades", "book", "candles", "status"] - -_Header = TypedDict("_Header", { - "event": Literal["subscribed"], - "channel": _Channel, - "chanId": int -}) - -#pylint: disable-next=inherit-non-class -class _Ticker(Ticker, _Header): - pass - -#pylint: disable-next=inherit-non-class -class _Trades(Trades, _Header): - pass - -#pylint: disable-next=inherit-non-class -class _Book(Book, _Header): - pass - -#pylint: disable-next=inherit-non-class -class _Candles(Candles, _Header): - pass - -#pylint: disable-next=inherit-non-class -class _Status(Status, _Header): - pass diff --git a/examples/websocket/public/order_book.py b/examples/websocket/public/order_book.py index 03d8c02..497e787 100644 --- a/examples/websocket/public/order_book.py +++ b/examples/websocket/public/order_book.py @@ -102,7 +102,7 @@ async def on_checksum(subscription: Book, value: int): print("Mismatch between local and remote checksums: " f"restarting book for symbol <{symbol}>...") - await bfx.wss.resubscribe(sub_id=subscription["subId"]) + await bfx.wss.resubscribe(sub_id=subscription["sub_id"]) order_book.cooldown[symbol] = True diff --git a/examples/websocket/public/raw_order_book.py b/examples/websocket/public/raw_order_book.py index c151dda..a08d9bb 100644 --- a/examples/websocket/public/raw_order_book.py +++ b/examples/websocket/public/raw_order_book.py @@ -102,7 +102,7 @@ async def on_checksum(subscription: Book, value: int): print("Mismatch between local and remote checksums: " f"restarting book for symbol <{symbol}>...") - await bfx.wss.resubscribe(sub_id=subscription["subId"]) + await bfx.wss.resubscribe(sub_id=subscription["sub_id"]) raw_order_book.cooldown[symbol] = True diff --git a/examples/websocket/public/ticker.py b/examples/websocket/public/ticker.py index 24c9463..253d467 100644 --- a/examples/websocket/public/ticker.py +++ b/examples/websocket/public/ticker.py @@ -10,7 +10,7 @@ bfx = Client(wss_host=PUB_WSS_HOST) @bfx.wss.on("t_ticker_update") def on_t_ticker_update(subscription: Ticker, data: TradingPairTicker): - print(f"Subscription with subId: {subscription['subId']}") + print(f"Subscription with sub_id: {subscription['sub_id']}") print(f"Data: {data}") From 9872adf60fbd6a082e826aa296b32762c749b934 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 8 Oct 2023 19:54:50 +0200 Subject: [PATCH 40/65] Fix type hinting in module bfxapi._utils.json_encoder. --- bfxapi/_utils/json_encoder.py | 26 +++++++++++-------- bfxapi/rest/endpoints/rest_auth_endpoints.py | 6 ++--- bfxapi/types/dataclasses.py | 18 ++++++------- .../websocket/_client/bfx_websocket_inputs.py | 6 ++--- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/bfxapi/_utils/json_encoder.py b/bfxapi/_utils/json_encoder.py index 9807480..27d2375 100644 --- a/bfxapi/_utils/json_encoder.py +++ b/bfxapi/_utils/json_encoder.py @@ -2,17 +2,21 @@ import json from decimal import Decimal -from typing import List, Dict, Union +from typing import \ + Union, List, Dict, \ + Any -JSON = Union[Dict[str, "JSON"], List["JSON"], bool, int, float, str, None] - -_CustomJSON = Union[Dict[str, "_CustomJSON"], List["_CustomJSON"], \ +_ExtJSON = Union[Dict[str, "_ExtJSON"], List["_ExtJSON"], \ bool, int, float, str, Decimal, None] -def _strip(dictionary: Dict) -> Dict: - return { key: value for key, value in dictionary.items() if value is not None } +_StrictJSON = Union[Dict[str, "_StrictJSON"], List["_StrictJSON"], \ + int, str, None] -def _convert_data_to_json(data: _CustomJSON) -> JSON: +def _clear(dictionary: Dict[str, Any]) -> Dict[str, Any]: + return { key: value for key, value in dictionary.items() \ + if value is not None } + +def _adapter(data: _ExtJSON) -> _StrictJSON: if isinstance(data, bool): return int(data) if isinstance(data, float): @@ -21,12 +25,12 @@ def _convert_data_to_json(data: _CustomJSON) -> JSON: return format(data, "f") if isinstance(data, list): - return [ _convert_data_to_json(sub_data) for sub_data in data ] + return [ _adapter(sub_data) for sub_data in data ] if isinstance(data, dict): - return _strip({ key: _convert_data_to_json(value) for key, value in data.items() }) + return _clear({ key: _adapter(value) for key, value in data.items() }) return data class JSONEncoder(json.JSONEncoder): - def encode(self, o: _CustomJSON) -> str: - return json.JSONEncoder.encode(self, _convert_data_to_json(o)) + def encode(self, o: _ExtJSON) -> str: + return super().encode(_adapter(o)) diff --git a/bfxapi/rest/endpoints/rest_auth_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py index ec4155e..661e17a 100644 --- a/bfxapi/rest/endpoints/rest_auth_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Tuple, Union, Literal, Optional +from typing import Dict, List, Tuple, Union, Literal, Optional, Any from decimal import Decimal @@ -22,8 +22,6 @@ from ...types import serializers from ...types.serializers import _Notification -from ..._utils.json_encoder import JSON - class RestAuthEndpoints(Middleware): def get_user_info(self) -> UserInfo: return serializers.UserInfo \ @@ -77,7 +75,7 @@ class RestAuthEndpoints(Middleware): cid: Optional[int] = None, flags: Optional[int] = 0, tif: Optional[str] = None, - meta: Optional[JSON] = None) -> Notification[Order]: + meta: Optional[Dict[str, Any]] = None) -> Notification[Order]: body = { "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, diff --git a/bfxapi/types/dataclasses.py b/bfxapi/types/dataclasses.py index d60cd38..937d786 100644 --- a/bfxapi/types/dataclasses.py +++ b/bfxapi/types/dataclasses.py @@ -5,8 +5,6 @@ from dataclasses import dataclass from .labeler import _Type, partial, compose -from .._utils.json_encoder import JSON - #region Dataclass definitions for types of public use @dataclass @@ -172,7 +170,7 @@ class PulseMessage(_Type): comments_disabled: int tags: List[str] attachments: List[str] - meta: List[JSON] + meta: List[Dict[str, Any]] likes: int profile: PulseProfile comments: int @@ -231,7 +229,7 @@ class LoginHistory(_Type): id: int time: int ip: str - extra_info: JSON + extra_info: Dict[str, Any] @dataclass class BalanceAvailable(_Type): @@ -260,7 +258,7 @@ class Order(_Type): hidden: int placed_id: int routing: str - meta: JSON + meta: Dict[str, Any] @dataclass class Position(_Type): @@ -280,7 +278,7 @@ class Position(_Type): type: int collateral: float collateral_min: float - meta: JSON + meta: Dict[str, Any] @dataclass class Trade(_Type): @@ -409,7 +407,7 @@ class Wallet(_Type): unsettled_interest: float available_balance: float last_change: str - trade_details: JSON + trade_details: Dict[str, Any] @dataclass class Transfer(_Type): @@ -486,7 +484,7 @@ class PositionClaim(_Type): pos_type: int collateral: str min_collateral: str - meta: JSON + meta: Dict[str, Any] @dataclass class PositionIncreaseInfo(_Type): @@ -547,7 +545,7 @@ class PositionAudit(_Type): type: int collateral: float collateral_min: float - meta: JSON + meta: Dict[str, Any] @dataclass class DerivativePositionCollateral(_Type): @@ -618,7 +616,7 @@ class InvoiceSubmission(_Type): pay_currency: str pool_currency: str address: str - ext: JSON + ext: Dict[str, Any] @compose(dataclass, partial) class Payment: diff --git a/bfxapi/websocket/_client/bfx_websocket_inputs.py b/bfxapi/websocket/_client/bfx_websocket_inputs.py index c84e9f5..c2d2884 100644 --- a/bfxapi/websocket/_client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/_client/bfx_websocket_inputs.py @@ -1,12 +1,10 @@ from typing import TYPE_CHECKING, Callable, Awaitable, \ - Tuple, List, Union, Optional, Any + Tuple, List, Dict, Union, Optional, Any if TYPE_CHECKING: from bfxapi.enums import \ OrderType, FundingOfferType - from bfxapi._utils.json_encoder import JSON - from decimal import Decimal class BfxWebSocketInputs: @@ -27,7 +25,7 @@ class BfxWebSocketInputs: cid: Optional[int] = None, flags: Optional[int] = 0, tif: Optional[str] = None, - meta: Optional["JSON"] = None) -> None: + meta: Optional[Dict[str, Any]] = None) -> None: await self.__handle_websocket_input("on", { "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, From de0ee54900d982c361d2bee648601d0c2bcd4d13 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Sun, 8 Oct 2023 21:49:36 +0200 Subject: [PATCH 41/65] Add new module bfxapi._utils.json_decoder. --- bfxapi/_utils/json_decoder.py | 13 ++ bfxapi/_utils/json_encoder.py | 8 +- .../rest/endpoints/rest_merchant_endpoints.py | 161 +++++++++--------- bfxapi/rest/middleware/middleware.py | 5 +- bfxapi/types/dataclasses.py | 4 +- .../websocket/_client/bfx_websocket_bucket.py | 17 +- 6 files changed, 107 insertions(+), 101 deletions(-) create mode 100644 bfxapi/_utils/json_decoder.py diff --git a/bfxapi/_utils/json_decoder.py b/bfxapi/_utils/json_decoder.py new file mode 100644 index 0000000..968258d --- /dev/null +++ b/bfxapi/_utils/json_decoder.py @@ -0,0 +1,13 @@ +from typing import Dict, Any + +import re, json + +def _to_snake_case(string: str) -> str: + return re.sub(r"(? Any: + return { _to_snake_case(key): value for key, value in data.items() } + +class JSONDecoder(json.JSONDecoder): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(object_hook=_object_hook, *args, **kwargs) diff --git a/bfxapi/_utils/json_encoder.py b/bfxapi/_utils/json_encoder.py index 27d2375..ac7883f 100644 --- a/bfxapi/_utils/json_encoder.py +++ b/bfxapi/_utils/json_encoder.py @@ -1,11 +1,11 @@ -import json - -from decimal import Decimal - from typing import \ Union, List, Dict, \ Any +import json + +from decimal import Decimal + _ExtJSON = Union[Dict[str, "_ExtJSON"], List["_ExtJSON"], \ bool, int, float, str, Decimal, None] diff --git a/bfxapi/rest/endpoints/rest_merchant_endpoints.py b/bfxapi/rest/endpoints/rest_merchant_endpoints.py index 8055e6c..d2ca4c9 100644 --- a/bfxapi/rest/endpoints/rest_merchant_endpoints.py +++ b/bfxapi/rest/endpoints/rest_merchant_endpoints.py @@ -1,46 +1,32 @@ -import re - -from typing import Callable, TypeVar, cast, \ - TypedDict, Dict, List, Union, Literal, Optional, Any +from typing import \ + TypedDict, Dict, List, \ + Union, Literal, Optional, \ + Any from decimal import Decimal -from ..middleware import Middleware +from bfxapi.rest.middleware import Middleware -from ..enums import MerchantSettingsKey +from bfxapi.rest.enums import MerchantSettingsKey -from ...types import \ - InvoiceSubmission, InvoicePage, InvoiceStats, \ - CurrencyConversion, MerchantDeposit, MerchantUnlinkedDeposit - -#region Defining methods to convert dictionary keys to snake_case and camelCase. - -T = TypeVar("T") - -_to_snake_case: Callable[[str], str] = lambda string: re.sub(r"(? T: - if isinstance(data, list): - return cast(T, [ _scheme(sub_data, adapter) for sub_data in data ]) - if isinstance(data, dict): - return cast(T, { adapter(key): _scheme(value, adapter) for key, value in data.items() }) - return data - -def _to_snake_case_keys(dictionary: T) -> T: - return _scheme(dictionary, _to_snake_case) - -def _to_camel_case_keys(dictionary: T) -> T: - return _scheme(dictionary, _to_camel_case) - -#endregion +from bfxapi.types import \ + InvoiceSubmission, \ + InvoicePage, \ + InvoiceStats, \ + CurrencyConversion, \ + MerchantDeposit, \ + MerchantUnlinkedDeposit _CustomerInfo = TypedDict("_CustomerInfo", { - "nationality": str, "resid_country": str, "resid_city": str, - "resid_zip_code": str, "resid_street": str, "resid_building_no": str, - "full_name": str, "email": str, "tos_accepted": bool + "nationality": str, + "resid_country": str, + "resid_city": str, + "resid_zip_code": str, + "resid_street": str, + "resid_building_no": str, + "full_name": str, + "email": str, + "tos_accepted": bool }) class RestMerchantEndpoints(Middleware): @@ -55,13 +41,13 @@ class RestMerchantEndpoints(Middleware): duration: Optional[int] = None, webhook: Optional[str] = None, redirect_url: Optional[str] = None) -> InvoiceSubmission: - body = _to_camel_case_keys({ - "amount": amount, "currency": currency, "order_id": order_id, - "customer_info": customer_info, "pay_currencies": pay_currencies, "duration": duration, - "webhook": webhook, "redirect_url": redirect_url - }) + body = { + "amount": amount, "currency": currency, "orderId": order_id, + "customerInfo": customer_info, "payCurrencies": pay_currencies, "duration": duration, + "webhook": webhook, "redirectUrl": redirect_url + } - data = _to_snake_case_keys(self._post("auth/w/ext/pay/invoice/create", body=body)) + data = self._post("auth/w/ext/pay/invoice/create", body=body) return InvoiceSubmission.parse(data) @@ -76,9 +62,9 @@ class RestMerchantEndpoints(Middleware): "limit": limit } - response = self._post("auth/r/ext/pay/invoices", body=body) + data = self._post("auth/r/ext/pay/invoices", body=body) - return [ InvoiceSubmission.parse(sub_data) for sub_data in _to_snake_case_keys(response) ] + return [ InvoiceSubmission.parse(sub_data) for sub_data in data ] def get_invoices_paginated(self, page: int = 1, @@ -91,13 +77,13 @@ class RestMerchantEndpoints(Middleware): crypto: Optional[List[str]] = None, id: Optional[str] = None, order_id: Optional[str] = None) -> InvoicePage: - body = _to_camel_case_keys({ - "page": page, "page_size": page_size, "sort": sort, - "sort_field": sort_field, "status": status, "fiat": fiat, - "crypto": crypto, "id": id, "order_id": order_id - }) + body = { + "page": page, "pageSize": page_size, "sort": sort, + "sortField": sort_field, "status": status, "fiat": fiat, + "crypto": crypto, "id": id, "orderId": order_id + } - data = _to_snake_case_keys(self._post("auth/r/ext/pay/invoices/paginated", body=body)) + data = self._post("auth/r/ext/pay/invoices/paginated", body=body) return InvoicePage.parse(data) @@ -105,13 +91,15 @@ class RestMerchantEndpoints(Middleware): status: Literal["CREATED", "PENDING", "COMPLETED", "EXPIRED"], format: str) -> List[InvoiceStats]: return [ InvoiceStats(**sub_data) for sub_data in \ - self._post("auth/r/ext/pay/invoice/stats/count", body={ "status": status, "format": format }) ] + self._post("auth/r/ext/pay/invoice/stats/count", \ + body={ "status": status, "format": format }) ] def get_invoice_earning_stats(self, currency: str, format: str) -> List[InvoiceStats]: return [ InvoiceStats(**sub_data) for sub_data in \ - self._post("auth/r/ext/pay/invoice/stats/earning", body={ "currency": currency, "format": format }) ] + self._post("auth/r/ext/pay/invoice/stats/earning", \ + body={ "currency": currency, "format": format }) ] def complete_invoice(self, id: str, @@ -119,45 +107,43 @@ class RestMerchantEndpoints(Middleware): *, deposit_id: Optional[int] = None, ledger_id: Optional[int] = None) -> InvoiceSubmission: - return InvoiceSubmission.parse(_to_snake_case_keys(self._post("auth/w/ext/pay/invoice/complete", body={ + body = { "id": id, "payCcy": pay_currency, "depositId": deposit_id, "ledgerId": ledger_id - }))) + } + + data = self._post("auth/w/ext/pay/invoice/complete", body=body) + + return InvoiceSubmission.parse(data) def expire_invoice(self, id: str) -> InvoiceSubmission: body = { "id": id } - response = self._post("auth/w/ext/pay/invoice/expire", body=body) - return InvoiceSubmission.parse(_to_snake_case_keys(response)) + + data = self._post("auth/w/ext/pay/invoice/expire", body=body) + + return InvoiceSubmission.parse(data) def get_currency_conversion_list(self) -> List[CurrencyConversion]: - return [ - CurrencyConversion( - base_currency=sub_data["baseCcy"], - convert_currency=sub_data["convertCcy"], - created=sub_data["created"] - ) for sub_data in self._post("auth/r/ext/pay/settings/convert/list") - ] + return [ CurrencyConversion(**sub_data) \ + for sub_data in self._post("auth/r/ext/pay/settings/convert/list") ] def add_currency_conversion(self, - base_currency: str, - convert_currency: str) -> bool: - return bool(self._post("auth/w/ext/pay/settings/convert/create", body={ - "baseCcy": base_currency, - "convertCcy": convert_currency - })) + base_ccy: str, + convert_ccy: str) -> bool: + return bool(self._post("auth/w/ext/pay/settings/convert/create", \ + body={ "baseCcy": base_ccy, "convertCcy": convert_ccy })) def remove_currency_conversion(self, - base_currency: str, - convert_currency: str) -> bool: - return bool(self._post("auth/w/ext/pay/settings/convert/remove", body={ - "baseCcy": base_currency, - "convertCcy": convert_currency - })) + base_ccy: str, + convert_ccy: str) -> bool: + return bool(self._post("auth/w/ext/pay/settings/convert/remove", \ + body={ "baseCcy": base_ccy, "convertCcy": convert_ccy })) def set_merchant_settings(self, key: MerchantSettingsKey, val: Any) -> bool: - return bool(self._post("auth/w/ext/pay/settings/set", body={ "key": key, "val": val })) + return bool(self._post("auth/w/ext/pay/settings/set", \ + body={ "key": key, "val": val })) def get_merchant_settings(self, key: MerchantSettingsKey) -> Any: return self._post("auth/r/ext/pay/settings/get", body={ "key": key }) @@ -167,19 +153,28 @@ class RestMerchantEndpoints(Middleware): def get_deposits(self, start: int, - end: int, + to: int, *, ccy: Optional[str] = None, unlinked: Optional[bool] = None) -> List[MerchantDeposit]: - body = { "from": start, "to": end, "ccy": ccy, "unlinked": unlinked } - response = self._post("auth/r/ext/pay/deposits", body=body) - return [ MerchantDeposit(**sub_data) for sub_data in _to_snake_case_keys(response) ] + body = { + "from": start, "to": to, "ccy": ccy, + "unlinked": unlinked + } + + data = self._post("auth/r/ext/pay/deposits", body=body) + + return [ MerchantDeposit(**sub_data) for sub_data in data ] def get_unlinked_deposits(self, ccy: str, *, start: Optional[int] = None, end: Optional[int] = None) -> List[MerchantUnlinkedDeposit]: - body = { "ccy": ccy, "start": start, "end": end } - response = self._post("/auth/r/ext/pay/deposits/unlinked", body=body) - return [ MerchantUnlinkedDeposit(**sub_data) for sub_data in _to_snake_case_keys(response) ] + body = { + "ccy": ccy, "start": start, "end": end + } + + data = self._post("/auth/r/ext/pay/deposits/unlinked", body=body) + + return [ MerchantUnlinkedDeposit(**sub_data) for sub_data in data ] diff --git a/bfxapi/rest/middleware/middleware.py b/bfxapi/rest/middleware/middleware.py index ae04b03..323990f 100644 --- a/bfxapi/rest/middleware/middleware.py +++ b/bfxapi/rest/middleware/middleware.py @@ -7,6 +7,7 @@ import time, hmac, hashlib, json, requests from ..enums import Error from ..exceptions import ResourceNotFound, RequestParametersError, InvalidAuthenticationCredentials, UnknownGenericError from ..._utils.json_encoder import JSONEncoder +from ..._utils.json_decoder import JSONDecoder if TYPE_CHECKING: from requests.sessions import _Params @@ -49,7 +50,7 @@ class Middleware: if response.status_code == HTTPStatus.NOT_FOUND: raise ResourceNotFound(f"No resources found at endpoint <{endpoint}>.") - data = response.json() + data = response.json(cls=JSONDecoder) if len(data) and data[0] == "error": if data[1] == Error.ERR_PARAMS: @@ -82,7 +83,7 @@ class Middleware: if response.status_code == HTTPStatus.NOT_FOUND: raise ResourceNotFound(f"No resources found at endpoint <{endpoint}>.") - data = response.json() + data = response.json(cls=JSONDecoder) if isinstance(data, list) and len(data) and data[0] == "error": if data[1] == Error.ERR_PARAMS: diff --git a/bfxapi/types/dataclasses.py b/bfxapi/types/dataclasses.py index 937d786..964f7fb 100644 --- a/bfxapi/types/dataclasses.py +++ b/bfxapi/types/dataclasses.py @@ -657,8 +657,8 @@ class InvoiceStats(_Type): @dataclass class CurrencyConversion(_Type): - base_currency: str - convert_currency: str + base_ccy: str + convert_ccy: str created: int @dataclass diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 3c42fdf..8d2aed7 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -7,16 +7,20 @@ import asyncio, json, uuid import websockets.client from pyee import EventEmitter +from bfxapi._utils.json_decoder import JSONDecoder from bfxapi.websocket._connection import Connection from bfxapi.websocket._handlers import PublicChannelsHandler + from bfxapi.websocket.subscriptions import Subscription from bfxapi.websocket.exceptions import FullBucketError + _CHECKSUM_FLAG_VALUE = 131_072 def _strip(message: Dict[str, Any], keys: List[str]) -> Dict[str, Any]: - return { key: message[key] for key in message if not key in keys } + return { key: value for key, value in message.items() \ + if not key in keys } class BfxWebSocketBucket(Connection): __MAXIMUM_SUBSCRIPTIONS_AMOUNT = 25 @@ -53,16 +57,9 @@ class BfxWebSocketBucket(Connection): self.__condition.notify(1) async for _message in self._websocket: - message = json.loads(_message) + message = json.loads(_message, cls=JSONDecoder) if isinstance(message, dict): - # I think there's a better way to do it... - if "chanId" in message: - message["chan_id"] = message.pop("chanId") - - if "subId" in message: - message["sub_id"] = message.pop("subId") - if message["event"] == "subscribed": self.__on_subscribed(message) elif message["event"] == "unsubscribed": @@ -83,7 +80,7 @@ class BfxWebSocketBucket(Connection): chan_id = cast(int, message["chan_id"]) subscription = cast(Subscription, _strip(message, \ - keys=["event", "chan_id", "pair", "currency"])) + keys=["chan_id", "event", "pair", "currency"])) self.__pendings = [ pending \ for pending in self.__pendings \ From 25881e77c8990c5895cdd95b6a02f04361a322ec Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 9 Oct 2023 16:25:46 +0200 Subject: [PATCH 42/65] Fix and rewrite some logic in class BfxWebSocketClient. --- .pylintrc | 2 +- bfxapi/client.py | 4 +- bfxapi/rest/middleware/middleware.py | 4 +- .../websocket/_client/bfx_websocket_bucket.py | 7 - .../websocket/_client/bfx_websocket_client.py | 131 ++++++++---------- 5 files changed, 62 insertions(+), 86 deletions(-) diff --git a/.pylintrc b/.pylintrc index 1f1a19b..3d6d4a5 100644 --- a/.pylintrc +++ b/.pylintrc @@ -24,7 +24,7 @@ max-line-length=120 expected-line-ending-format=LF [BASIC] -good-names=t,f,id,ip,on,pl,tf,A,B,C,D,E,F +good-names=t,f,id,ip,on,pl,tf,to,A,B,C,D,E,F [TYPECHECK] generated-members=websockets diff --git a/bfxapi/client.py b/bfxapi/client.py index 3d7f3e6..21fdafe 100644 --- a/bfxapi/client.py +++ b/bfxapi/client.py @@ -3,7 +3,7 @@ from typing import \ from bfxapi._utils.logging import ColorLogger -from bfxapi.exceptions import IncompleteCredentialError +from bfxapi._exceptions import IncompleteCredentialError from bfxapi.rest import BfxRestInterface from bfxapi.websocket import BfxWebSocketClient @@ -22,7 +22,7 @@ class Client: rest_host: str = REST_HOST, wss_host: str = WSS_HOST, filters: Optional[List[str]] = None, - timeout: Optional[float] = 60 * 15, + timeout: Optional[int] = 60 * 15, log_filename: Optional[str] = None, log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" ) -> None: diff --git a/bfxapi/rest/middleware/middleware.py b/bfxapi/rest/middleware/middleware.py index 323990f..cf434e5 100644 --- a/bfxapi/rest/middleware/middleware.py +++ b/bfxapi/rest/middleware/middleware.py @@ -5,7 +5,7 @@ from http import HTTPStatus import time, hmac, hashlib, json, requests from ..enums import Error -from ..exceptions import ResourceNotFound, RequestParametersError, InvalidAuthenticationCredentials, UnknownGenericError +from ..exceptions import ResourceNotFound, RequestParametersError, InvalidCredentialError, UnknownGenericError from ..._utils.json_encoder import JSONEncoder from ..._utils.json_decoder import JSONDecoder @@ -91,7 +91,7 @@ class Middleware: f"following parameter error: <{data[2]}>") if data[1] == Error.ERR_AUTH_FAIL: - raise InvalidAuthenticationCredentials("Cannot authenticate with given API-KEY and API-SECRET.") + raise InvalidCredentialError("Cannot authenticate with given API-KEY and API-SECRET.") if data[1] is None or data[1] == Error.ERR_UNK or data[1] == Error.ERR_GENERIC: raise UnknownGenericError("The server replied to the request with " \ diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 8d2aed7..7e2ada1 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -13,9 +13,6 @@ from bfxapi.websocket._handlers import PublicChannelsHandler from bfxapi.websocket.subscriptions import Subscription -from bfxapi.websocket.exceptions import FullBucketError - - _CHECKSUM_FLAG_VALUE = 131_072 def _strip(message: Dict[str, Any], keys: List[str]) -> Dict[str, Any]: @@ -111,10 +108,6 @@ class BfxWebSocketBucket(Connection): channel: str, sub_id: Optional[str] = None, **kwargs: Any) -> None: - if self.is_full: - raise FullBucketError("The bucket is full: " + \ - "can't subscribe to any other channel.") - subscription: Dict[str, Any] = \ { **kwargs, "event": "subscribe", "channel": channel } diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 1518e35..48d525e 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -1,24 +1,24 @@ from typing import \ - TYPE_CHECKING, TypedDict, List, \ - Dict, Optional, Any, \ - no_type_check + TypedDict, List, Dict, \ + Optional, Any, no_type_check from logging import Logger + from datetime import datetime from socket import gaierror +from asyncio import Task import \ traceback, json, asyncio, \ hmac, hashlib, random, \ websockets +import websockets.client + from websockets.exceptions import \ ConnectionClosedError, \ InvalidStatusCode -from websockets.legacy.client import \ - connect as _websockets__connect - from bfxapi._utils.json_encoder import JSONEncoder from bfxapi.websocket._connection import Connection @@ -26,24 +26,22 @@ from bfxapi.websocket._handlers import AuthEventsHandler from bfxapi.websocket._event_emitter import BfxEventEmitter from bfxapi.websocket.exceptions import \ - InvalidAuthenticationCredentials, \ + InvalidCredentialError, \ ReconnectionTimeoutError, \ - OutdatedClientVersion, \ + VersionMismatchError, \ ZeroConnectionsError, \ - UnknownChannelError + UnknownChannelError, \ + UnknownSubscriptionError -from bfxapi.websocket._client.bfx_websocket_bucket import BfxWebSocketBucket +from .bfx_websocket_bucket import BfxWebSocketBucket -from bfxapi.websocket._client.bfx_websocket_inputs import BfxWebSocketInputs +from .bfx_websocket_inputs import BfxWebSocketInputs -if TYPE_CHECKING: - from asyncio import Task +_Credentials = TypedDict("_Credentials", \ + { "api_key": str, "api_secret": str, "filters": Optional[List[str]] }) - _Credentials = TypedDict("_Credentials", \ - { "api_key": str, "api_secret": str, "filters": Optional[List[str]] }) - - _Reconnection = TypedDict("_Reconnection", - { "attempts": int, "reason": str, "timestamp": datetime }) +_Reconnection = TypedDict("_Reconnection", + { "attempts": int, "reason": str, "timestamp": datetime }) _DEFAULT_LOGGER = Logger("bfxapi.websocket._client", level=0) @@ -72,22 +70,18 @@ class _Delay: self.__backoff_delay = _Delay.__BACKOFF_MIN class BfxWebSocketClient(Connection): - VERSION = 2 - - MAXIMUM_CONNECTIONS_AMOUNT = 20 - def __init__(self, host: str, *, - credentials: Optional["_Credentials"] = None, - timeout: Optional[float] = 60 * 15, + credentials: Optional[_Credentials] = None, + timeout: Optional[int] = 60 * 15, logger: Logger = _DEFAULT_LOGGER) -> None: super().__init__(host) self.__credentials, self.__timeout, self.__logger = \ credentials, timeout, logger - self.__buckets: Dict[BfxWebSocketBucket, Optional["Task"]] = { } + self.__buckets: Dict[BfxWebSocketBucket, Optional[Task]] = { } self.__reconnection: Optional[_Reconnection] = None @@ -113,32 +107,14 @@ class BfxWebSocketClient(Connection): def inputs(self) -> BfxWebSocketInputs: return self.__inputs - def run(self, connections: int = 5) -> None: - return asyncio.run(self.start(connections)) - - async def start(self, connections: int = 5) -> None: - if connections == 0: - self.__logger.info("With connections set to 0 it will not be possible to subscribe to any " \ - "public channel. Attempting a subscription will cause a ZeroConnectionsError to be thrown.") - - if connections > BfxWebSocketClient.MAXIMUM_CONNECTIONS_AMOUNT: - self.__logger.warning(f"It is not safe to use more than {BfxWebSocketClient.MAXIMUM_CONNECTIONS_AMOUNT} " \ - f"buckets from the same connection ({connections} in use), the server could momentarily " \ - "block the client with <429 Too Many Requests>.") - - for _ in range(connections): - _bucket = BfxWebSocketBucket( \ - self._host, self.__event_emitter) - - self.__buckets.update({ _bucket: None }) - - await self.__connect() + def run(self) -> None: + return asyncio.run(self.start()) #pylint: disable-next=too-many-branches - async def __connect(self) -> None: + async def start(self) -> None: _delay = _Delay(backoff_factor=1.618) - _sleep: Optional["Task"] = None + _sleep: Optional[Task] = None def _on_timeout(): if not self.open: @@ -158,19 +134,20 @@ class BfxWebSocketClient(Connection): from None try: - await self.__connection() + await self.__connect() except (ConnectionClosedError, InvalidStatusCode, gaierror) as error: - async def _cancel(task: "Task") -> None: + async def _cancel(task: Task) -> None: task.cancel() try: await task - except (ConnectionClosedError, gaierror) as _e: + except (ConnectionClosedError, InvalidStatusCode, gaierror) as _e: if type(error) is not type(_e) or error.args != _e.args: raise _e except asyncio.CancelledError: pass + # pylint: disable-next=consider-using-dict-items for bucket in self.__buckets: if task := self.__buckets[bucket]: self.__buckets[bucket] = None @@ -186,7 +163,7 @@ class BfxWebSocketClient(Connection): self.__logger.info("WSS server is about to restart, clients need " \ "to reconnect (server sent 20051). Reconnection attempt in progress...") - if self.__timeout is not None: + if self.__timeout: asyncio.get_event_loop().call_later( self.__timeout, _on_timeout) @@ -214,8 +191,8 @@ class BfxWebSocketClient(Connection): break - async def __connection(self) -> None: - async with _websockets__connect(self._host) as websocket: + async def __connect(self) -> None: + async with websockets.client.connect(self._host) as websocket: if self.__reconnection: self.__logger.info(f"_Reconnection attempt successful (no.{self.__reconnection['attempts']}): The " \ f"client has been offline for a total of {datetime.now() - self.__reconnection['timestamp']} " \ @@ -225,11 +202,9 @@ class BfxWebSocketClient(Connection): self._websocket = websocket - self.__buckets = { - bucket: asyncio.create_task(_c) - for bucket in self.__buckets - if (_c := bucket.start()) - } + for bucket in self.__buckets: + self.__buckets[bucket] = \ + asyncio.create_task(bucket.start()) if len(self.__buckets) == 0 or \ (await asyncio.gather(*[bucket.wait() for bucket in self.__buckets])): @@ -241,29 +216,31 @@ class BfxWebSocketClient(Connection): await self._websocket.send(authentication) - async for message in self._websocket: - message = json.loads(message) + async for _message in self._websocket: + message = json.loads(_message) if isinstance(message, dict): if message["event"] == "info" and "version" in message: - if BfxWebSocketClient.VERSION != message["version"]: - raise OutdatedClientVersion("Mismatch between the client version and the server " \ - "version. Update the library to the latest version to continue (client version: " \ - f"{BfxWebSocketClient.VERSION}, server version: {message['version']}).") + if message["version"] != 2: + raise VersionMismatchError("Mismatch between the client and the server version: " + \ + "please update bitfinex-api-py to the latest version to resolve this error " + \ + f"(client version: 2, server version: {message['version']}).") elif message["event"] == "info" and message["code"] == 20051: - code, reason = 1012, "Stop/Restart WebSocket Server (please reconnect)." - rcvd = websockets.frames.Close(code=code, reason=reason) + rcvd = websockets.frames.Close( \ + 1012, "Stop/Restart WebSocket Server (please reconnect).") + raise ConnectionClosedError(rcvd=rcvd, sent=None) elif message["event"] == "auth": if message["status"] != "OK": - raise InvalidAuthenticationCredentials( - "Cannot authenticate with given API-KEY and API-SECRET.") + raise InvalidCredentialError("Cannot authenticate " + \ + "with given API-KEY and API-SECRET.") self.__event_emitter.emit("authenticated", message) self._authentication = True elif message["event"] == "error": - self.__event_emitter.emit("wss-error", message["code"], message["msg"]) + self.__event_emitter.emit("wss-error", \ + message["code"], message["msg"]) if isinstance(message, list) and \ message[0] == 0 and message[1] != Connection._HEARTBEAT: @@ -294,14 +271,20 @@ class BfxWebSocketClient(Connection): @Connection.require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: for bucket in self.__buckets: - if bucket.has(sub_id=sub_id): - await bucket.unsubscribe(sub_id=sub_id) + if bucket.has(sub_id): + return await bucket.unsubscribe(sub_id) + + raise UnknownSubscriptionError("Unable to find " + \ + f"a subscription with sub_id <{sub_id}>.") @Connection.require_websocket_connection async def resubscribe(self, sub_id: str) -> None: for bucket in self.__buckets: - if bucket.has(sub_id=sub_id): - await bucket.resubscribe(sub_id=sub_id) + if bucket.has(sub_id): + return await bucket.resubscribe(sub_id) + + raise UnknownSubscriptionError("Unable to find " + \ + f"a subscription with sub_id <{sub_id}>.") @Connection.require_websocket_connection async def close(self, code: int = 1000, reason: str = str()) -> None: @@ -323,7 +306,7 @@ class BfxWebSocketClient(Connection): @Connection.require_websocket_authentication async def __handle_websocket_input(self, event: str, data: Any) -> None: - await self._websocket.send(json.dumps(\ + await self._websocket.send(json.dumps( \ [ 0, event, None, data], cls=JSONEncoder)) @no_type_check From 378e89b5049af416b2a381de662e5bf44b14e93f Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 9 Oct 2023 16:27:04 +0200 Subject: [PATCH 43/65] Fix small bug in module bfxapi.exceptions. --- bfxapi/{exceptions.py => _exceptions.py} | 9 +++-- bfxapi/rest/exceptions.py | 28 +++----------- bfxapi/websocket/exceptions.py | 48 ++++++++---------------- 3 files changed, 25 insertions(+), 60 deletions(-) rename bfxapi/{exceptions.py => _exceptions.py} (63%) diff --git a/bfxapi/exceptions.py b/bfxapi/_exceptions.py similarity index 63% rename from bfxapi/exceptions.py rename to bfxapi/_exceptions.py index b636119..e408bd6 100644 --- a/bfxapi/exceptions.py +++ b/bfxapi/_exceptions.py @@ -1,7 +1,3 @@ -__all__ = [ - "BfxBaseException", -] - class BfxBaseException(Exception): """ Base class for every custom exception in bfxapi/rest/exceptions.py and bfxapi/websocket/exceptions.py. @@ -11,3 +7,8 @@ class IncompleteCredentialError(BfxBaseException): """ This error indicates an incomplete credential object (missing api-key or api-secret). """ + +class InvalidCredentialError(BfxBaseException): + """ + This error indicates that the user has provided incorrect credentials (API-KEY and API-SECRET) for authentication. + """ diff --git a/bfxapi/rest/exceptions.py b/bfxapi/rest/exceptions.py index 0c506d1..ab7001c 100644 --- a/bfxapi/rest/exceptions.py +++ b/bfxapi/rest/exceptions.py @@ -1,35 +1,17 @@ -from ..exceptions import BfxBaseException +# pylint: disable-next=wildcard-import,unused-wildcard-import +from bfxapi._exceptions import * -__all__ = [ - "BfxRestException", - - "ResourceNotFound", - "RequestParametersError", - "ResourceNotFound", - "InvalidAuthenticationCredentials" -] - -class BfxRestException(BfxBaseException): - """ - Base class for all custom exceptions in bfxapi/rest/exceptions.py. - """ - -class ResourceNotFound(BfxRestException): +class ResourceNotFound(BfxBaseException): """ This error indicates a failed HTTP request to a non-existent resource. """ -class RequestParametersError(BfxRestException): +class RequestParametersError(BfxBaseException): """ This error indicates that there are some invalid parameters sent along with an HTTP request. """ -class InvalidAuthenticationCredentials(BfxRestException): - """ - This error indicates that the user has provided incorrect credentials (API-KEY and API-SECRET) for authentication. - """ - -class UnknownGenericError(BfxRestException): +class UnknownGenericError(BfxBaseException): """ This error indicates an undefined problem processing an HTTP request sent to the APIs. """ diff --git a/bfxapi/websocket/exceptions.py b/bfxapi/websocket/exceptions.py index c23da60..1720122 100644 --- a/bfxapi/websocket/exceptions.py +++ b/bfxapi/websocket/exceptions.py @@ -1,65 +1,47 @@ -from ..exceptions import BfxBaseException +# pylint: disable-next=wildcard-import,unused-wildcard-import +from bfxapi._exceptions import * -__all__ = [ - "BfxWebSocketException", - - "ConnectionNotOpen", - "FullBucketError", - "ZeroConnectionsError", - "ReconnectionTimeoutError", - "ActionRequiresAuthentication", - "InvalidAuthenticationCredentials", - "UnknownChannelError", - "UnknownEventError", - "OutdatedClientVersion" -] - -class BfxWebSocketException(BfxBaseException): - """ - Base class for all custom exceptions in bfxapi/websocket/exceptions.py. - """ - -class ConnectionNotOpen(BfxWebSocketException): +class ConnectionNotOpen(BfxBaseException): """ This error indicates an attempt to communicate via websocket before starting the connection with the servers. """ -class FullBucketError(BfxWebSocketException): +class FullBucketError(BfxBaseException): """ Thrown when a user attempts a subscription but all buckets are full. """ -class ZeroConnectionsError(BfxWebSocketException): +class ZeroConnectionsError(BfxBaseException): """ This error indicates an attempt to subscribe to a public channel while the number of connections is 0. """ -class ReconnectionTimeoutError(BfxWebSocketException): +class ReconnectionTimeoutError(BfxBaseException): """ This error indicates that the connection has been offline for too long without being able to reconnect. """ -class ActionRequiresAuthentication(BfxWebSocketException): +class ActionRequiresAuthentication(BfxBaseException): """ This error indicates an attempt to access a protected resource without logging in first. """ -class InvalidAuthenticationCredentials(BfxWebSocketException): - """ - This error indicates that the user has provided incorrect credentials (API-KEY and API-SECRET) for authentication. - """ - -class UnknownChannelError(BfxWebSocketException): +class UnknownChannelError(BfxBaseException): """ Thrown when a user attempts to subscribe to an unknown channel. """ -class UnknownEventError(BfxWebSocketException): +class UnknownSubscriptionError(BfxBaseException): + """ + Thrown when a user attempts to reference an unknown subscription. + """ + +class UnknownEventError(BfxBaseException): """ Thrown when a user attempts to add a listener for an unknown event. """ -class OutdatedClientVersion(BfxWebSocketException): +class VersionMismatchError(BfxBaseException): """ This error indicates a mismatch between the client version and the server WSS version. """ From 122d692684dd0b48a9a4724ef186c35ae8a65f4f Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 13 Oct 2023 05:43:42 +0200 Subject: [PATCH 44/65] Rewrite all logic regarding connection multiplexing. --- README.md | 22 -------------- .../websocket/_client/bfx_websocket_client.py | 29 ++++++++++++------- bfxapi/websocket/exceptions.py | 5 ---- 3 files changed, 18 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 004b346..38caf3d 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,6 @@ _Revoke your API-KEYs and API-SECRETs immediately if you think they might have b ### Advanced features * [Using custom notifications](#using-custom-notifications) -* [Setting up connection multiplexing](#setting-up-connection-multiplexing) ### Examples * [Creating a new order](#creating-a-new-order) @@ -264,27 +263,6 @@ def on_notification(notification: Notification[Any]): print(notification.data) # { "foo": 1 } ``` -## Setting up connection multiplexing - -`BfxWebSocketClient::run` and `BfxWebSocketClient::start` accept a `connections` argument: -```python -bfx.wss.run(connections=3) -``` - -`connections` indicates the number of connections to run concurrently (through connection multiplexing). - -Each of these connections can handle up to 25 subscriptions to public channels. \ -So, using `N` connections will allow the client to handle at most `N * 25` subscriptions. \ -You should always use the minimum number of connections necessary to handle all the subscriptions that will be made. - -For example, if you know that your application will subscribe to 75 public channels, 75 / 25 = 3 connections will be enough to handle all the subscriptions. - -The default number of connections is 5; therefore, if the `connections` argument is not given, the client will be able to handle a maximum of 25 * 5 = 125 subscriptions. - -Keep in mind that using a large number of connections could slow down the client performance. - -The use of more than 20 connections is not recommended. - # Examples ## Creating a new order diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 48d525e..5063946 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -29,7 +29,6 @@ from bfxapi.websocket.exceptions import \ InvalidCredentialError, \ ReconnectionTimeoutError, \ VersionMismatchError, \ - ZeroConnectionsError, \ UnknownChannelError, \ UnknownSubscriptionError @@ -246,27 +245,35 @@ class BfxWebSocketClient(Connection): message[0] == 0 and message[1] != Connection._HEARTBEAT: self.__handler.handle(message[1], message[2]) + async def __new_bucket(self) -> BfxWebSocketBucket: + bucket = BfxWebSocketBucket( \ + self._host, self.__event_emitter) + + self.__buckets[bucket] = asyncio \ + .create_task(bucket.start()) + + await bucket.wait() + + return bucket + @Connection.require_websocket_connection async def subscribe(self, channel: str, sub_id: Optional[str] = None, **kwargs: Any) -> None: - if len(self.__buckets) == 0: - raise ZeroConnectionsError("Unable to subscribe: " \ - "the number of connections must be greater than 0.") - if not channel in ["ticker", "trades", "book", "candles", "status"]: raise UnknownChannelError("Available channels are: " + \ "ticker, trades, book, candles and status.") - _buckets = list(self.__buckets.keys()) + for bucket in self.__buckets: + if not bucket.is_full: + return await bucket.subscribe( \ + channel, sub_id, **kwargs) - counters = [ bucket.count for bucket in _buckets ] + bucket = await self.__new_bucket() - index = counters.index(min(counters)) - - await _buckets[index] \ - .subscribe(channel, sub_id, **kwargs) + return await bucket.subscribe( \ + channel, sub_id, **kwargs) @Connection.require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: diff --git a/bfxapi/websocket/exceptions.py b/bfxapi/websocket/exceptions.py index 1720122..e7019ab 100644 --- a/bfxapi/websocket/exceptions.py +++ b/bfxapi/websocket/exceptions.py @@ -11,11 +11,6 @@ class FullBucketError(BfxBaseException): Thrown when a user attempts a subscription but all buckets are full. """ -class ZeroConnectionsError(BfxBaseException): - """ - This error indicates an attempt to subscribe to a public channel while the number of connections is 0. - """ - class ReconnectionTimeoutError(BfxBaseException): """ This error indicates that the connection has been offline for too long without being able to reconnect. From 374b632c6c319c24a07569be52365975e9783f32 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 13 Oct 2023 17:03:36 +0200 Subject: [PATCH 45/65] Add pause/resume logic in class BfxWebSocketClient. --- .../websocket/_client/bfx_websocket_client.py | 11 ++++-- .../_event_emitter/bfx_event_emitter.py | 37 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 5063946..dff5ec6 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -78,7 +78,9 @@ class BfxWebSocketClient(Connection): super().__init__(host) self.__credentials, self.__timeout, self.__logger = \ - credentials, timeout, logger + credentials, \ + timeout, \ + logger self.__buckets: Dict[BfxWebSocketBucket, Optional[Task]] = { } @@ -99,15 +101,16 @@ class BfxWebSocketClient(Connection): stack_trace = traceback.format_exception( \ type(exception), exception, exception.__traceback__) - self.__logger.critical( \ - header + "\n" + str().join(stack_trace)[:-1]) + self.__logger.critical(header + "\n" + \ + str().join(stack_trace)[:-1]) @property def inputs(self) -> BfxWebSocketInputs: return self.__inputs def run(self) -> None: - return asyncio.run(self.start()) + return asyncio.get_event_loop() \ + .run_until_complete(self.start()) #pylint: disable-next=too-many-branches async def start(self) -> None: diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py index e53eee8..79a7959 100644 --- a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -9,9 +9,9 @@ from pyee.asyncio import AsyncIOEventEmitter from bfxapi.websocket.exceptions import UnknownEventError _ONCE_PER_CONNECTION = [ - "open", "authenticated", "disconnected", - "order_snapshot", "position_snapshot", "funding_offer_snapshot", - "funding_credit_snapshot", "funding_loan_snapshot", "wallet_snapshot" + "open", "authenticated", "order_snapshot", + "position_snapshot", "funding_offer_snapshot", "funding_credit_snapshot", + "funding_loan_snapshot", "wallet_snapshot" ] _ONCE_PER_SUBSCRIPTION = [ @@ -21,25 +21,26 @@ _ONCE_PER_SUBSCRIPTION = [ ] _COMMON = [ - "error", "wss-error", "t_ticker_update", - "f_ticker_update", "t_trade_execution", "t_trade_execution_update", - "f_trade_execution", "f_trade_execution_update", "t_book_update", - "f_book_update", "t_raw_book_update", "f_raw_book_update", - "candles_update", "derivatives_status_update", "liquidation_feed_update", - "order_new", "order_update", "order_cancel", - "position_new", "position_update", "position_close", - "funding_offer_new", "funding_offer_update", "funding_offer_cancel", - "funding_credit_new", "funding_credit_update", "funding_credit_close", - "funding_loan_new", "funding_loan_update", "funding_loan_close", - "trade_execution", "trade_execution_update", "wallet_update", - "notification", "on-req-notification", "ou-req-notification", - "oc-req-notification", "fon-req-notification", "foc-req-notification" + "error", "wss-error", "disconnected", + "t_ticker_update", "f_ticker_update", "t_trade_execution", + "t_trade_execution_update", "f_trade_execution", "f_trade_execution_update", + "t_book_update", "f_book_update", "t_raw_book_update", + "f_raw_book_update", "candles_update", "derivatives_status_update", + "liquidation_feed_update", "order_new", "order_update", + "order_cancel", "position_new", "position_update", + "position_close", "funding_offer_new", "funding_offer_update", + "funding_offer_cancel", "funding_credit_new", "funding_credit_update", + "funding_credit_close", "funding_loan_new", "funding_loan_update", + "funding_loan_close", "trade_execution", "trade_execution_update", + "wallet_update", "notification", "on-req-notification", + "ou-req-notification", "oc-req-notification", "fon-req-notification", + "foc-req-notification" ] class BfxEventEmitter(AsyncIOEventEmitter): _EVENTS = _ONCE_PER_CONNECTION + \ - _ONCE_PER_SUBSCRIPTION + \ - _COMMON + _ONCE_PER_SUBSCRIPTION + \ + _COMMON def __init__(self, loop: Optional[AbstractEventLoop] = None) -> None: super().__init__(loop) From e5ec94b75781275d939817f94111e242136c5aee Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Fri, 13 Oct 2023 17:38:25 +0200 Subject: [PATCH 46/65] Remove wss-event event from BfxWebSocketClient and BfxWebSocketBucket. --- .../websocket/_client/bfx_websocket_bucket.py | 3 --- .../websocket/_client/bfx_websocket_client.py | 3 --- .../_event_emitter/bfx_event_emitter.py | 27 +++++++++---------- examples/websocket/auth/submit_order.py | 4 --- examples/websocket/auth/wallets.py | 4 --- .../websocket/public/derivatives_status.py | 4 --- examples/websocket/public/order_book.py | 4 --- examples/websocket/public/raw_order_book.py | 4 --- examples/websocket/public/trades.py | 4 --- 9 files changed, 13 insertions(+), 44 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 7e2ada1..2fb1670 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -64,9 +64,6 @@ class BfxWebSocketBucket(Connection): chan_id = cast(int, message["chan_id"]) del self.__subscriptions[chan_id] - elif message["event"] == "error": - self.__event_emitter.emit("wss-error", \ - message["code"], message["msg"]) if isinstance(message, list): if (chan_id := cast(int, message[0])) and \ diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index dff5ec6..1250b85 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -240,9 +240,6 @@ class BfxWebSocketClient(Connection): self.__event_emitter.emit("authenticated", message) self._authentication = True - elif message["event"] == "error": - self.__event_emitter.emit("wss-error", \ - message["code"], message["msg"]) if isinstance(message, list) and \ message[0] == 0 and message[1] != Connection._HEARTBEAT: diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py index 79a7959..31ee983 100644 --- a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -21,20 +21,19 @@ _ONCE_PER_SUBSCRIPTION = [ ] _COMMON = [ - "error", "wss-error", "disconnected", - "t_ticker_update", "f_ticker_update", "t_trade_execution", - "t_trade_execution_update", "f_trade_execution", "f_trade_execution_update", - "t_book_update", "f_book_update", "t_raw_book_update", - "f_raw_book_update", "candles_update", "derivatives_status_update", - "liquidation_feed_update", "order_new", "order_update", - "order_cancel", "position_new", "position_update", - "position_close", "funding_offer_new", "funding_offer_update", - "funding_offer_cancel", "funding_credit_new", "funding_credit_update", - "funding_credit_close", "funding_loan_new", "funding_loan_update", - "funding_loan_close", "trade_execution", "trade_execution_update", - "wallet_update", "notification", "on-req-notification", - "ou-req-notification", "oc-req-notification", "fon-req-notification", - "foc-req-notification" + "error", "disconnected", "t_ticker_update", + "f_ticker_update", "t_trade_execution", "t_trade_execution_update", + "f_trade_execution", "f_trade_execution_update", "t_book_update", + "f_book_update", "t_raw_book_update", "f_raw_book_update", + "candles_update", "derivatives_status_update", "liquidation_feed_update", + "order_new", "order_update", "order_cancel", + "position_new", "position_update", "position_close", + "funding_offer_new", "funding_offer_update", "funding_offer_cancel", + "funding_credit_new", "funding_credit_update", "funding_credit_close", + "funding_loan_new", "funding_loan_update", "funding_loan_close", + "trade_execution", "trade_execution_update", "wallet_update", + "notification", "on-req-notification", "ou-req-notification", + "oc-req-notification", "fon-req-notification", "foc-req-notification" ] class BfxEventEmitter(AsyncIOEventEmitter): diff --git a/examples/websocket/auth/submit_order.py b/examples/websocket/auth/submit_order.py index 0c2d03b..a65935a 100644 --- a/examples/websocket/auth/submit_order.py +++ b/examples/websocket/auth/submit_order.py @@ -12,10 +12,6 @@ bfx = Client( api_secret=os.getenv("BFX_API_SECRET") ) -@bfx.wss.on("wss-error") -def on_wss_error(code: Error, msg: str): - print(code, msg) - @bfx.wss.on("authenticated") async def on_authenticated(event): print(f"Authentication: {event}") diff --git a/examples/websocket/auth/wallets.py b/examples/websocket/auth/wallets.py index 057ad29..a8d40ed 100644 --- a/examples/websocket/auth/wallets.py +++ b/examples/websocket/auth/wallets.py @@ -14,10 +14,6 @@ bfx = Client( filters=["wallet"] ) -@bfx.wss.on("wss-error") -def on_wss_error(code: Error, msg: str): - print(code, msg) - @bfx.wss.on("wallet_snapshot") def on_wallet_snapshot(wallets: List[Wallet]): for wallet in wallets: diff --git a/examples/websocket/public/derivatives_status.py b/examples/websocket/public/derivatives_status.py index d55c492..982d3cb 100644 --- a/examples/websocket/public/derivatives_status.py +++ b/examples/websocket/public/derivatives_status.py @@ -12,10 +12,6 @@ bfx = Client(wss_host=PUB_WSS_HOST) def on_derivatives_status_update(subscription: Status, data: DerivativesStatus): print(f"{subscription}:", data) -@bfx.wss.on("wss-error") -def on_wss_error(code: Error, msg: str): - print(code, msg) - @bfx.wss.on("open") async def on_open(): await bfx.wss.subscribe(Channel.STATUS, key="deriv:tBTCF0:USTF0") diff --git a/examples/websocket/public/order_book.py b/examples/websocket/public/order_book.py index 497e787..5201ed8 100644 --- a/examples/websocket/public/order_book.py +++ b/examples/websocket/public/order_book.py @@ -70,10 +70,6 @@ order_book = OrderBook(symbols=SYMBOLS) bfx = Client(wss_host=PUB_WSS_HOST) -@bfx.wss.on("wss-error") -def on_wss_error(code: Error, msg: str): - print(code, msg) - @bfx.wss.on("open") async def on_open(): for symbol in SYMBOLS: diff --git a/examples/websocket/public/raw_order_book.py b/examples/websocket/public/raw_order_book.py index a08d9bb..dedd291 100644 --- a/examples/websocket/public/raw_order_book.py +++ b/examples/websocket/public/raw_order_book.py @@ -70,10 +70,6 @@ raw_order_book = RawOrderBook(symbols=SYMBOLS) bfx = Client(wss_host=PUB_WSS_HOST) -@bfx.wss.on("wss-error") -def on_wss_error(code: Error, msg: str): - print(code, msg) - @bfx.wss.on("open") async def on_open(): for symbol in SYMBOLS: diff --git a/examples/websocket/public/trades.py b/examples/websocket/public/trades.py index e079904..79dc71e 100644 --- a/examples/websocket/public/trades.py +++ b/examples/websocket/public/trades.py @@ -16,10 +16,6 @@ def on_candles_update(_sub: Candles, candle: Candle): def on_t_trade_execution(_sub: Trades, trade: TradingPairTrade): print(f"New trade: {trade}") -@bfx.wss.on("wss-error") -def on_wss_error(code: Error, msg: str): - print(code, msg) - @bfx.wss.on("open") async def on_open(): await bfx.wss.subscribe(Channel.CANDLES, key="trade:1m:tBTCUSD") From 133db74a72babc069eef2e312d16f1ba9d9d6342 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 16 Oct 2023 04:45:47 +0200 Subject: [PATCH 47/65] Add automatic deletion for buckets that reach zero subscriptions (e.g. after a call to BfxWebSocketClient::unsubscribe). --- bfxapi/websocket/_client/bfx_websocket_bucket.py | 14 ++++++++------ bfxapi/websocket/_client/bfx_websocket_client.py | 15 ++++++++++++++- bfxapi/websocket/exceptions.py | 5 +++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 2fb1670..753bf73 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -44,6 +44,11 @@ class BfxWebSocketBucket(Connection): return self.count == \ BfxWebSocketBucket.__MAXIMUM_SUBSCRIPTIONS_AMOUNT + @property + def ids(self) -> List[str]: + return [ pending["subId"] for pending in self.__pendings ] + \ + [ subscription["sub_id"] for subscription in self.__subscriptions.values() ] + async def start(self) -> None: async with websockets.client.connect(self._host) as websocket: self._websocket = websocket @@ -59,11 +64,6 @@ class BfxWebSocketBucket(Connection): if isinstance(message, dict): if message["event"] == "subscribed": self.__on_subscribed(message) - elif message["event"] == "unsubscribed": - if message["status"] == "OK": - chan_id = cast(int, message["chan_id"]) - - del self.__subscriptions[chan_id] if isinstance(message, list): if (chan_id := cast(int, message[0])) and \ @@ -117,12 +117,14 @@ class BfxWebSocketBucket(Connection): @Connection.require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: - for chan_id, subscription in self.__subscriptions.items(): + for chan_id, subscription in list(self.__subscriptions.items()): if subscription["sub_id"] == sub_id: unsubscription = { "event": "unsubscribe", "chanId": chan_id } + del self.__subscriptions[chan_id] + await self._websocket.send(message = \ json.dumps(unsubscription)) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 1250b85..82d2801 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -30,7 +30,8 @@ from bfxapi.websocket.exceptions import \ ReconnectionTimeoutError, \ VersionMismatchError, \ UnknownChannelError, \ - UnknownSubscriptionError + UnknownSubscriptionError, \ + SubIdError from .bfx_websocket_bucket import BfxWebSocketBucket @@ -265,6 +266,11 @@ class BfxWebSocketClient(Connection): raise UnknownChannelError("Available channels are: " + \ "ticker, trades, book, candles and status.") + for bucket in self.__buckets: + if sub_id in bucket.ids: + raise SubIdError("sub_id must be " + \ + "unique for all subscriptions.") + for bucket in self.__buckets: if not bucket.is_full: return await bucket.subscribe( \ @@ -277,8 +283,15 @@ class BfxWebSocketClient(Connection): @Connection.require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: + # pylint: disable-next=consider-using-dict-items for bucket in self.__buckets: if bucket.has(sub_id): + if bucket.count == 1: + del self.__buckets[bucket] + + return await bucket.close( \ + code=1001, reason="Going Away") + return await bucket.unsubscribe(sub_id) raise UnknownSubscriptionError("Unable to find " + \ diff --git a/bfxapi/websocket/exceptions.py b/bfxapi/websocket/exceptions.py index e7019ab..e662cbf 100644 --- a/bfxapi/websocket/exceptions.py +++ b/bfxapi/websocket/exceptions.py @@ -40,3 +40,8 @@ class VersionMismatchError(BfxBaseException): """ This error indicates a mismatch between the client version and the server WSS version. """ + +class SubIdError(BfxBaseException): + """ + Thrown when a user attempts to open more than one subscription using the same sub_id. + """ From ddce83be0c56c3064ba90a36e28717f64ad6905b Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Mon, 16 Oct 2023 05:59:24 +0200 Subject: [PATCH 48/65] Apply some refactoring to sub-package bfxapi.websocket. --- .../websocket/_client/bfx_websocket_bucket.py | 8 +-- .../websocket/_client/bfx_websocket_client.py | 45 ++++---------- bfxapi/websocket/_connection.py | 61 ++++++++++++++----- .../_handlers/auth_events_handler.py | 43 +++++++------ .../_handlers/public_channels_handler.py | 41 ++++++------- 5 files changed, 102 insertions(+), 96 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_bucket.py b/bfxapi/websocket/_client/bfx_websocket_bucket.py index 753bf73..8d83385 100644 --- a/bfxapi/websocket/_client/bfx_websocket_bucket.py +++ b/bfxapi/websocket/_client/bfx_websocket_bucket.py @@ -100,7 +100,7 @@ class BfxWebSocketBucket(Connection): await self._websocket.send(json.dumps( \ { "event": "conf", "flags": sum(flags) })) - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def subscribe(self, channel: str, sub_id: Optional[str] = None, @@ -115,7 +115,7 @@ class BfxWebSocketBucket(Connection): await self._websocket.send(message = \ json.dumps(subscription)) - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: for chan_id, subscription in list(self.__subscriptions.items()): if subscription["sub_id"] == sub_id: @@ -128,7 +128,7 @@ class BfxWebSocketBucket(Connection): await self._websocket.send(message = \ json.dumps(unsubscription)) - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def resubscribe(self, sub_id: str) -> None: for subscription in self.__subscriptions.values(): if subscription["sub_id"] == sub_id: @@ -136,7 +136,7 @@ class BfxWebSocketBucket(Connection): await self.subscribe(**subscription) - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def close(self, code: int = 1000, reason: str = str()) -> None: await self._websocket.close(code, reason) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index 82d2801..ad33985 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -1,17 +1,16 @@ from typing import \ TypedDict, List, Dict, \ - Optional, Any, no_type_check + Optional, Any from logging import Logger - from datetime import datetime from socket import gaierror + from asyncio import Task import \ traceback, json, asyncio, \ - hmac, hashlib, random, \ - websockets + random, websockets import websockets.client @@ -214,8 +213,8 @@ class BfxWebSocketClient(Connection): self.__event_emitter.emit("open") if self.__credentials: - authentication = BfxWebSocketClient. \ - __build_authentication_message(**self.__credentials) + authentication = Connection. \ + _get_authentication_message(**self.__credentials) await self._websocket.send(authentication) @@ -235,7 +234,7 @@ class BfxWebSocketClient(Connection): raise ConnectionClosedError(rcvd=rcvd, sent=None) elif message["event"] == "auth": if message["status"] != "OK": - raise InvalidCredentialError("Cannot authenticate " + \ + raise InvalidCredentialError("Can't authenticate " + \ "with given API-KEY and API-SECRET.") self.__event_emitter.emit("authenticated", message) @@ -257,7 +256,7 @@ class BfxWebSocketClient(Connection): return bucket - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def subscribe(self, channel: str, sub_id: Optional[str] = None, @@ -281,7 +280,7 @@ class BfxWebSocketClient(Connection): return await bucket.subscribe( \ channel, sub_id, **kwargs) - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def unsubscribe(self, sub_id: str) -> None: # pylint: disable-next=consider-using-dict-items for bucket in self.__buckets: @@ -297,7 +296,7 @@ class BfxWebSocketClient(Connection): raise UnknownSubscriptionError("Unable to find " + \ f"a subscription with sub_id <{sub_id}>.") - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def resubscribe(self, sub_id: str) -> None: for bucket in self.__buckets: if bucket.has(sub_id): @@ -306,7 +305,7 @@ class BfxWebSocketClient(Connection): raise UnknownSubscriptionError("Unable to find " + \ f"a subscription with sub_id <{sub_id}>.") - @Connection.require_websocket_connection + @Connection._require_websocket_connection async def close(self, code: int = 1000, reason: str = str()) -> None: for bucket in self.__buckets: await bucket.close(code=code, reason=reason) @@ -315,7 +314,7 @@ class BfxWebSocketClient(Connection): await self._websocket.close( \ code=code, reason=reason) - @Connection.require_websocket_authentication + @Connection._require_websocket_authentication async def notify(self, info: Any, message_id: Optional[int] = None, @@ -324,30 +323,10 @@ class BfxWebSocketClient(Connection): json.dumps([ 0, "n", message_id, { "type": "ucm-test", "info": info, **kwargs } ])) - @Connection.require_websocket_authentication + @Connection._require_websocket_authentication async def __handle_websocket_input(self, event: str, data: Any) -> None: await self._websocket.send(json.dumps( \ [ 0, event, None, data], cls=JSONEncoder)) - @no_type_check def on(self, event, f = None): return self.__event_emitter.on(event, f=f) - - @staticmethod - def __build_authentication_message(api_key: str, - api_secret: str, - filters: Optional[List[str]] = None) -> str: - message: Dict[str, Any] = \ - { "event": "auth", "filter": filters, "apiKey": api_key } - - message["authNonce"] = round(datetime.now().timestamp() * 1_000_000) - - message["authPayload"] = f"AUTH{message['authNonce']}" - - message["authSig"] = hmac.new( - key=api_secret.encode("utf8"), - msg=message["authPayload"].encode("utf8"), - digestmod=hashlib.sha384 - ).hexdigest() - - return json.dumps(message) diff --git a/bfxapi/websocket/_connection.py b/bfxapi/websocket/_connection.py index 971bdd0..779488e 100644 --- a/bfxapi/websocket/_connection.py +++ b/bfxapi/websocket/_connection.py @@ -1,18 +1,24 @@ from typing import \ - TYPE_CHECKING, TypeVar, Callable, \ - Awaitable, Optional, Any, \ - cast + TypeVar, Callable, Awaitable, \ + List, Dict, Optional, \ + Any, cast -from abc import ABC, abstractmethod +# pylint: disable-next=wrong-import-order +from typing_extensions import \ + ParamSpec, Concatenate -from typing_extensions import ParamSpec, Concatenate +from abc import \ + ABC, abstractmethod + +from datetime import datetime + +import hmac, hashlib, json + +from websockets.client import WebSocketClientProtocol from bfxapi.websocket.exceptions import \ ConnectionNotOpen, ActionRequiresAuthentication -if TYPE_CHECKING: - from websockets.client import WebSocketClientProtocol - _S = TypeVar("_S", bound="Connection") _R = TypeVar("_R") @@ -27,7 +33,7 @@ class Connection(ABC): self._authentication: bool = False - self.__protocol: Optional["WebSocketClientProtocol"] = None + self.__protocol: Optional[WebSocketClientProtocol] = None @property def open(self) -> bool: @@ -39,11 +45,11 @@ class Connection(ABC): return self._authentication @property - def _websocket(self) -> "WebSocketClientProtocol": - return cast("WebSocketClientProtocol", self.__protocol) + def _websocket(self) -> WebSocketClientProtocol: + return cast(WebSocketClientProtocol, self.__protocol) @_websocket.setter - def _websocket(self, protocol: "WebSocketClientProtocol") -> None: + def _websocket(self, protocol: WebSocketClientProtocol) -> None: self.__protocol = protocol @abstractmethod @@ -51,9 +57,9 @@ class Connection(ABC): ... @staticmethod - def require_websocket_connection( + def _require_websocket_connection( function: Callable[Concatenate[_S, _P], Awaitable[_R]] - ) -> Callable[Concatenate[_S, _P], Awaitable["_R"]]: + ) -> Callable[Concatenate[_S, _P], Awaitable[_R]]: async def wrapper(self: _S, *args: Any, **kwargs: Any) -> _R: if self.open: return await function(self, *args, **kwargs) @@ -63,7 +69,7 @@ class Connection(ABC): return wrapper @staticmethod - def require_websocket_authentication( + def _require_websocket_authentication( function: Callable[Concatenate[_S, _P], Awaitable[_R]] ) -> Callable[Concatenate[_S, _P], Awaitable[_R]]: async def wrapper(self: _S, *args: Any, **kwargs: Any) -> _R: @@ -71,8 +77,31 @@ class Connection(ABC): raise ActionRequiresAuthentication("To perform this action you need to " \ "authenticate using your API_KEY and API_SECRET.") - internal = Connection.require_websocket_connection(function) + internal = Connection._require_websocket_connection(function) return await internal(self, *args, **kwargs) return wrapper + + @staticmethod + def _get_authentication_message( + api_key: str, + api_secret: str, + filters: Optional[List[str]] = None + ) -> str: + message: Dict[str, Any] = \ + { "event": "auth", "filter": filters, "apiKey": api_key } + + message["authNonce"] = round(datetime.now().timestamp() * 1_000_000) + + message["authPayload"] = f"AUTH{message['authNonce']}" + + auth_sig = hmac.new( + key=api_secret.encode("utf8"), + msg=message["authPayload"].encode("utf8"), + digestmod=hashlib.sha384 + ) + + message["authSig"] = auth_sig.hexdigest() + + return json.dumps(message) diff --git a/bfxapi/websocket/_handlers/auth_events_handler.py b/bfxapi/websocket/_handlers/auth_events_handler.py index 18940f2..d93364d 100644 --- a/bfxapi/websocket/_handlers/auth_events_handler.py +++ b/bfxapi/websocket/_handlers/auth_events_handler.py @@ -1,15 +1,14 @@ -from typing import TYPE_CHECKING, \ +from typing import \ Dict, Tuple, Any +from pyee.base import EventEmitter + from bfxapi.types import serializers from bfxapi.types.serializers import _Notification -if TYPE_CHECKING: - from bfxapi.types.dataclasses import \ - Order, FundingOffer - - from pyee.base import EventEmitter +from bfxapi.types.dataclasses import \ + Order, FundingOffer class AuthEventsHandler: __ABBREVIATIONS = { @@ -23,24 +22,24 @@ class AuthEventsHandler: "flc": "funding_loan_close", "ws": "wallet_snapshot", "wu": "wallet_update" } - def __init__(self, event_emitter: "EventEmitter") -> None: - self.__event_emitter = event_emitter + __SERIALIZERS: Dict[Tuple[str, ...], serializers._Serializer] = { + ("os", "on", "ou", "oc"): serializers.Order, + ("ps", "pn", "pu", "pc"): serializers.Position, + ("te", "tu"): serializers.Trade, + ("fos", "fon", "fou", "foc"): serializers.FundingOffer, + ("fcs", "fcn", "fcu", "fcc"): serializers.FundingCredit, + ("fls", "fln", "flu", "flc"): serializers.FundingLoan, + ("ws", "wu"): serializers.Wallet + } - self.__serializers: Dict[Tuple[str, ...], serializers._Serializer] = { - ("os", "on", "ou", "oc",): serializers.Order, - ("ps", "pn", "pu", "pc",): serializers.Position, - ("te", "tu"): serializers.Trade, - ("fos", "fon", "fou", "foc",): serializers.FundingOffer, - ("fcs", "fcn", "fcu", "fcc",): serializers.FundingCredit, - ("fls", "fln", "flu", "flc",): serializers.FundingLoan, - ("ws", "wu",): serializers.Wallet - } + def __init__(self, event_emitter: EventEmitter) -> None: + self.__event_emitter = event_emitter def handle(self, abbrevation: str, stream: Any) -> None: if abbrevation == "n": return self.__notification(stream) - for abbrevations, serializer in self.__serializers.items(): + for abbrevations, serializer in AuthEventsHandler.__SERIALIZERS.items(): if abbrevation in abbrevations: event = AuthEventsHandler.__ABBREVIATIONS[abbrevation] @@ -57,12 +56,12 @@ class AuthEventsHandler: serializer: _Notification = _Notification[None](serializer=None) - if stream[1] == "on-req" or stream[1] == "ou-req" or stream[1] == "oc-req": + if stream[1] in ("on-req", "ou-req", "oc-req"): event, serializer = f"{stream[1]}-notification", \ - _Notification["Order"](serializer=serializers.Order) + _Notification[Order](serializer=serializers.Order) - if stream[1] == "fon-req" or stream[1] == "foc-req": + if stream[1] in ("fon-req", "foc-req"): event, serializer = f"{stream[1]}-notification", \ - _Notification["FundingOffer"](serializer=serializers.FundingOffer) + _Notification[FundingOffer](serializer=serializers.FundingOffer) self.__event_emitter.emit(event, serializer.parse(*stream)) diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index 597f16c..9e46989 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -1,28 +1,27 @@ from typing import \ - TYPE_CHECKING, List, Any, \ - cast + List, Any, cast + +from pyee.base import EventEmitter from bfxapi.types import serializers -if TYPE_CHECKING: - from bfxapi.websocket.subscriptions import Subscription, \ - Ticker, Trades, Book, Candles, Status - - from pyee.base import EventEmitter +from bfxapi.websocket.subscriptions import \ + Subscription, Ticker, Trades, \ + Book, Candles, Status _CHECKSUM = "cs" class PublicChannelsHandler: - def __init__(self, event_emitter: "EventEmitter") -> None: + def __init__(self, event_emitter: EventEmitter) -> None: self.__event_emitter = event_emitter - def handle(self, subscription: "Subscription", stream: List[Any]) -> None: + def handle(self, subscription: Subscription, stream: List[Any]) -> None: if subscription["channel"] == "ticker": - self.__ticker_channel_handler(cast("Ticker", subscription), stream) + self.__ticker_channel_handler(cast(Ticker, subscription), stream) elif subscription["channel"] == "trades": - self.__trades_channel_handler(cast("Trades", subscription), stream) + self.__trades_channel_handler(cast(Trades, subscription), stream) elif subscription["channel"] == "book": - subscription = cast("Book", subscription) + subscription = cast(Book, subscription) if stream[0] == _CHECKSUM: self.__checksum_handler(subscription, stream[1]) @@ -32,11 +31,11 @@ class PublicChannelsHandler: else: self.__raw_book_channel_handler(subscription, stream) elif subscription["channel"] == "candles": - self.__candles_channel_handler(cast("Candles", subscription), stream) + self.__candles_channel_handler(cast(Candles, subscription), stream) elif subscription["channel"] == "status": - self.__status_channel_handler(cast("Status", subscription), stream) + self.__status_channel_handler(cast(Status, subscription), stream) - def __ticker_channel_handler(self, subscription: "Ticker", stream: List[Any]): + def __ticker_channel_handler(self, subscription: Ticker, stream: List[Any]): if subscription["symbol"].startswith("t"): return self.__event_emitter.emit("t_ticker_update", subscription, \ serializers.TradingPairTicker.parse(*stream[0])) @@ -45,7 +44,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("f_ticker_update", subscription, \ serializers.FundingCurrencyTicker.parse(*stream[0])) - def __trades_channel_handler(self, subscription: "Trades", stream: List[Any]): + def __trades_channel_handler(self, subscription: Trades, stream: List[Any]): if (event := stream[0]) and event in [ "te", "tu", "fte", "ftu" ]: events = { "te": "t_trade_execution", "tu": "t_trade_execution_update", \ "fte": "f_trade_execution", "ftu": "f_trade_execution_update" } @@ -68,7 +67,7 @@ class PublicChannelsHandler: [ serializers.FundingCurrencyTrade.parse(*sub_stream) \ for sub_stream in stream[0] ]) - def __book_channel_handler(self, subscription: "Book", stream: List[Any]): + def __book_channel_handler(self, subscription: Book, stream: List[Any]): if subscription["symbol"].startswith("t"): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): return self.__event_emitter.emit("t_book_snapshot", subscription, \ @@ -87,7 +86,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("f_book_update", subscription, \ serializers.FundingCurrencyBook.parse(*stream[0])) - def __raw_book_channel_handler(self, subscription: "Book", stream: List[Any]): + def __raw_book_channel_handler(self, subscription: Book, stream: List[Any]): if subscription["symbol"].startswith("t"): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): return self.__event_emitter.emit("t_raw_book_snapshot", subscription, \ @@ -106,7 +105,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("f_raw_book_update", subscription, \ serializers.FundingCurrencyRawBook.parse(*stream[0])) - def __candles_channel_handler(self, subscription: "Candles", stream: List[Any]): + def __candles_channel_handler(self, subscription: Candles, stream: List[Any]): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): return self.__event_emitter.emit("candles_snapshot", subscription, \ [ serializers.Candle.parse(*sub_stream) \ @@ -115,7 +114,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("candles_update", subscription, \ serializers.Candle.parse(*stream[0])) - def __status_channel_handler(self, subscription: "Status", stream: List[Any]): + def __status_channel_handler(self, subscription: Status, stream: List[Any]): if subscription["key"].startswith("deriv:"): return self.__event_emitter.emit("derivatives_status_update", subscription, \ serializers.DerivativesStatus.parse(*stream[0])) @@ -124,6 +123,6 @@ class PublicChannelsHandler: return self.__event_emitter.emit("liquidation_feed_update", subscription, \ serializers.Liquidation.parse(*stream[0][0])) - def __checksum_handler(self, subscription: "Book", value: int): + def __checksum_handler(self, subscription: Book, value: int): return self.__event_emitter.emit( \ "checksum", subscription, value & 0xFFFFFFFF) From ac50f8f884e706d81adccd6ce07d716d3af76892 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Wed, 25 Oct 2023 05:52:55 +0200 Subject: [PATCH 49/65] Fix and rewrite module bfx_websocket_inputs in bfxapi.websocket._client. --- .../websocket/_client/bfx_websocket_inputs.py | 58 ++++++++++--------- bfxapi/websocket/_connection.py | 4 ++ 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_inputs.py b/bfxapi/websocket/_client/bfx_websocket_inputs.py index c2d2884..f14d32d 100644 --- a/bfxapi/websocket/_client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/_client/bfx_websocket_inputs.py @@ -1,52 +1,54 @@ -from typing import TYPE_CHECKING, Callable, Awaitable, \ - Tuple, List, Dict, Union, Optional, Any +from typing import \ + Callable, Awaitable, Tuple, \ + List, Union, Optional, \ + Any -if TYPE_CHECKING: - from bfxapi.enums import \ - OrderType, FundingOfferType +from decimal import Decimal - from decimal import Decimal +from bfxapi.enums import \ + OrderType, \ + FundingOfferType + +_Handler = Callable[[str, Any], Awaitable[None]] class BfxWebSocketInputs: - def __init__(self, handle_websocket_input: Callable[[str, Any], Awaitable[None]]) -> None: + def __init__(self, handle_websocket_input: _Handler) -> None: self.__handle_websocket_input = handle_websocket_input async def submit_order(self, - type: "OrderType", + type: OrderType, symbol: str, - amount: Union["Decimal", float, str], + amount: Union[str, float, Decimal], + price: Union[str, float, Decimal], *, - price: Optional[Union["Decimal", float, str]] = None, lev: Optional[int] = None, - price_trailing: Optional[Union["Decimal", float, str]] = None, - price_aux_limit: Optional[Union["Decimal", float, str]] = None, - price_oco_stop: Optional[Union["Decimal", float, str]] = None, + price_trailing: Optional[Union[str, float, Decimal]] = None, + price_aux_limit: Optional[Union[str, float, Decimal]] = None, + price_oco_stop: Optional[Union[str, float, Decimal]] = None, gid: Optional[int] = None, cid: Optional[int] = None, - flags: Optional[int] = 0, - tif: Optional[str] = None, - meta: Optional[Dict[str, Any]] = None) -> None: + flags: Optional[int] = None, + tif: Optional[str] = None) -> None: await self.__handle_websocket_input("on", { "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, "price_aux_limit": price_aux_limit, "price_oco_stop": price_oco_stop, "gid": gid, "cid": cid, "flags": flags, "tif": tif, - "meta": meta }) async def update_order(self, id: int, *, - amount: Optional[Union["Decimal", float, str]] = None, - price: Optional[Union["Decimal", float, str]] = None, + amount: Optional[Union[str, float, Decimal]] = None, + price: Optional[Union[str, float, Decimal]] = None, cid: Optional[int] = None, cid_date: Optional[str] = None, gid: Optional[int] = None, - flags: Optional[int] = 0, + flags: Optional[int] = None, lev: Optional[int] = None, - delta: Optional[Union["Decimal", float, str]] = None, - price_aux_limit: Optional[Union["Decimal", float, str]] = None, - price_trailing: Optional[Union["Decimal", float, str]] = None, + delta: Optional[Union[str, float, Decimal]] = None, + price_aux_limit: Optional[Union[str, float, Decimal]] = None, + price_trailing: Optional[Union[str, float, Decimal]] = None, tif: Optional[str] = None) -> None: await self.__handle_websocket_input("ou", { "id": id, "amount": amount, "price": price, @@ -69,7 +71,7 @@ class BfxWebSocketInputs: ids: Optional[List[int]] = None, cids: Optional[List[Tuple[int, str]]] = None, gids: Optional[List[int]] = None, - all: bool = False) -> None: + all: Optional[bool] = None) -> None: await self.__handle_websocket_input("oc_multi", { "ids": ids, "cids": cids, "gids": gids, "all": all @@ -77,13 +79,13 @@ class BfxWebSocketInputs: #pylint: disable-next=too-many-arguments async def submit_funding_offer(self, - type: "FundingOfferType", + type: FundingOfferType, symbol: str, - amount: Union["Decimal", float, str], - rate: Union["Decimal", float, str], + amount: Union[str, float, Decimal], + rate: Union[str, float, Decimal], period: int, *, - flags: Optional[int] = 0) -> None: + flags: Optional[int] = None) -> None: await self.__handle_websocket_input("fon", { "type": type, "symbol": symbol, "amount": amount, "rate": rate, "period": period, "flags": flags diff --git a/bfxapi/websocket/_connection.py b/bfxapi/websocket/_connection.py index 779488e..d43339e 100644 --- a/bfxapi/websocket/_connection.py +++ b/bfxapi/websocket/_connection.py @@ -10,6 +10,8 @@ from typing_extensions import \ from abc import \ ABC, abstractmethod +from functools import wraps + from datetime import datetime import hmac, hashlib, json @@ -60,6 +62,7 @@ class Connection(ABC): def _require_websocket_connection( function: Callable[Concatenate[_S, _P], Awaitable[_R]] ) -> Callable[Concatenate[_S, _P], Awaitable[_R]]: + @wraps(function) async def wrapper(self: _S, *args: Any, **kwargs: Any) -> _R: if self.open: return await function(self, *args, **kwargs) @@ -72,6 +75,7 @@ class Connection(ABC): def _require_websocket_authentication( function: Callable[Concatenate[_S, _P], Awaitable[_R]] ) -> Callable[Concatenate[_S, _P], Awaitable[_R]]: + @wraps(function) async def wrapper(self: _S, *args: Any, **kwargs: Any) -> _R: if not self.authentication: raise ActionRequiresAuthentication("To perform this action you need to " \ From 8e915e42eb93de613242aac817a4aa7a87d20a63 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 05:09:10 +0200 Subject: [PATCH 50/65] Improve fidelity to pylint's standard rules. --- .pylintrc | 16 +++------- bfxapi/client.py | 7 ++--- bfxapi/rest/endpoints/rest_auth_endpoints.py | 1 + .../rest/endpoints/rest_merchant_endpoints.py | 1 + .../rest/endpoints/rest_public_endpoints.py | 1 + bfxapi/types/labeler.py | 20 ++++++------- bfxapi/types/notification.py | 2 +- .../websocket/_client/bfx_websocket_client.py | 30 +++++++++++-------- .../_handlers/auth_events_handler.py | 7 ++--- .../_handlers/public_channels_handler.py | 7 +++++ 10 files changed, 49 insertions(+), 43 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3d6d4a5..c616eb1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,28 +3,20 @@ py-version=3.8.0 [MESSAGES CONTROL] disable= - multiple-imports, missing-docstring, - logging-not-lazy, - logging-fstring-interpolation, + multiple-imports, too-few-public-methods, - too-many-public-methods, - too-many-instance-attributes, - dangerous-default-value, - inconsistent-return-statements, - -[SIMILARITIES] -min-similarity-lines=6 + too-many-instance-attributes [VARIABLES] -allowed-redefined-builtins=type,dir,id,all,format,len +allowed-redefined-builtins=all,dir,format,id,len,type [FORMAT] max-line-length=120 expected-line-ending-format=LF [BASIC] -good-names=t,f,id,ip,on,pl,tf,to,A,B,C,D,E,F +good-names=f,t,id,ip,on,pl,tf,to,A,B,C,D,E,F [TYPECHECK] generated-members=websockets diff --git a/bfxapi/client.py b/bfxapi/client.py index 21fdafe..d45b925 100644 --- a/bfxapi/client.py +++ b/bfxapi/client.py @@ -1,5 +1,5 @@ from typing import \ - TYPE_CHECKING, List, Literal, Optional + TYPE_CHECKING, List, Optional from bfxapi._utils.logging import ColorLogger @@ -23,8 +23,7 @@ class Client: wss_host: str = WSS_HOST, filters: Optional[List[str]] = None, timeout: Optional[int] = 60 * 15, - log_filename: Optional[str] = None, - log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" + log_filename: Optional[str] = None ) -> None: credentials: Optional["_Credentials"] = None @@ -40,7 +39,7 @@ class Client: self.rest = BfxRestInterface(rest_host, api_key, api_secret) - logger = ColorLogger("bfxapi", level=log_level) + logger = ColorLogger("bfxapi", level="INFO") if log_filename: logger.register(filename=log_filename) diff --git a/bfxapi/rest/endpoints/rest_auth_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py index 661e17a..3dc0885 100644 --- a/bfxapi/rest/endpoints/rest_auth_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -22,6 +22,7 @@ from ...types import serializers from ...types.serializers import _Notification +#pylint: disable-next=too-many-public-methods class RestAuthEndpoints(Middleware): def get_user_info(self) -> UserInfo: return serializers.UserInfo \ diff --git a/bfxapi/rest/endpoints/rest_merchant_endpoints.py b/bfxapi/rest/endpoints/rest_merchant_endpoints.py index d2ca4c9..d201a10 100644 --- a/bfxapi/rest/endpoints/rest_merchant_endpoints.py +++ b/bfxapi/rest/endpoints/rest_merchant_endpoints.py @@ -148,6 +148,7 @@ class RestMerchantEndpoints(Middleware): def get_merchant_settings(self, key: MerchantSettingsKey) -> Any: return self._post("auth/r/ext/pay/settings/get", body={ "key": key }) + #pylint: disable-next=dangerous-default-value def list_merchant_settings(self, keys: List[MerchantSettingsKey] = []) -> Dict[MerchantSettingsKey, Any]: return self._post("auth/r/ext/pay/settings/list", body={ "keys": keys }) diff --git a/bfxapi/rest/endpoints/rest_public_endpoints.py b/bfxapi/rest/endpoints/rest_public_endpoints.py index e1c20ff..4401057 100644 --- a/bfxapi/rest/endpoints/rest_public_endpoints.py +++ b/bfxapi/rest/endpoints/rest_public_endpoints.py @@ -17,6 +17,7 @@ from ...types import \ from ...types import serializers +#pylint: disable-next=too-many-public-methods class RestPublicEndpoints(Middleware): def conf(self, config: Config) -> Any: return self._get(f"conf/{config}")[0] diff --git a/bfxapi/types/labeler.py b/bfxapi/types/labeler.py index 52ac497..8ad5896 100644 --- a/bfxapi/types/labeler.py +++ b/bfxapi/types/labeler.py @@ -34,8 +34,8 @@ class _Type: class _Serializer(Generic[T]): def __init__(self, name: str, klass: Type[_Type], labels: List[str], - *, flat: bool = False, ignore: List[str] = [ "_PLACEHOLDER" ]): - self.name, self.klass, self.__labels, self.__flat, self.__ignore = name, klass, labels, flat, ignore + *, flat: bool = False): + self.name, self.klass, self.__labels, self.__flat = name, klass, labels, flat def _serialize(self, *args: Any) -> Iterable[Tuple[str, Any]]: if self.__flat: @@ -46,14 +46,14 @@ class _Serializer(Generic[T]): "arguments should contain the same amount of elements.") for index, label in enumerate(self.__labels): - if label not in self.__ignore: + if label != "_PLACEHOLDER": yield label, args[index] def parse(self, *values: Any) -> T: return cast(T, self.klass(**dict(self._serialize(*values)))) def get_labels(self) -> List[str]: - return [ label for label in self.__labels if label not in self.__ignore ] + return [ label for label in self.__labels if label != "_PLACEHOLDER" ] @classmethod def __flatten(cls, array: List[Any]) -> List[Any]: @@ -68,8 +68,8 @@ class _Serializer(Generic[T]): class _RecursiveSerializer(_Serializer, Generic[T]): def __init__(self, name: str, klass: Type[_Type], labels: List[str], *, serializers: Dict[str, _Serializer[Any]], - flat: bool = False, ignore: List[str] = [ "_PLACEHOLDER" ]): - super().__init__(name, klass, labels, flat=flat, ignore=ignore) + flat: bool = False): + super().__init__(name, klass, labels, flat=flat) self.serializers = serializers @@ -83,14 +83,14 @@ class _RecursiveSerializer(_Serializer, Generic[T]): return cast(T, self.klass(**serialization)) def generate_labeler_serializer(name: str, klass: Type[T], labels: List[str], - *, flat: bool = False, ignore: List[str] = [ "_PLACEHOLDER" ] + *, flat: bool = False ) -> _Serializer[T]: return _Serializer[T](name, klass, labels, \ - flat=flat, ignore=ignore) + flat=flat) def generate_recursive_serializer(name: str, klass: Type[T], labels: List[str], *, serializers: Dict[str, _Serializer[Any]], - flat: bool = False, ignore: List[str] = [ "_PLACEHOLDER" ] + flat: bool = False ) -> _RecursiveSerializer[T]: return _RecursiveSerializer[T](name, klass, labels, \ - serializers=serializers, flat=flat, ignore=ignore) + serializers=serializers, flat=flat) diff --git a/bfxapi/types/notification.py b/bfxapi/types/notification.py index ae02259..add3175 100644 --- a/bfxapi/types/notification.py +++ b/bfxapi/types/notification.py @@ -18,7 +18,7 @@ class _Notification(_Serializer, Generic[T]): __LABELS = [ "mts", "type", "message_id", "_PLACEHOLDER", "data", "code", "status", "text" ] def __init__(self, serializer: Optional[_Serializer] = None, is_iterable: bool = False): - super().__init__("Notification", Notification, _Notification.__LABELS, ignore = [ "_PLACEHOLDER" ]) + super().__init__("Notification", Notification, _Notification.__LABELS) self.serializer, self.is_iterable = serializer, is_iterable diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index ad33985..a8831c0 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -3,9 +3,9 @@ from typing import \ Optional, Any from logging import Logger + from datetime import datetime from socket import gaierror - from asyncio import Task import \ @@ -68,6 +68,7 @@ class _Delay: def reset(self) -> None: self.__backoff_delay = _Delay.__BACKOFF_MIN +#pylint: disable-next=too-many-instance-attributes class BfxWebSocketClient(Connection): def __init__(self, host: str, @@ -101,6 +102,7 @@ class BfxWebSocketClient(Connection): stack_trace = traceback.format_exception( \ type(exception), exception, exception.__traceback__) + #pylint: disable-next=logging-not-lazy self.__logger.critical(header + "\n" + \ str().join(stack_trace)[:-1]) @@ -158,12 +160,11 @@ class BfxWebSocketClient(Connection): if isinstance(error, ConnectionClosedError) and error.code in (1006, 1012): if error.code == 1006: - self.__logger.error("Connection lost: no close frame " \ - "received or sent (1006). Trying to reconnect...") + self.__logger.error("Connection lost: trying to reconnect...") if error.code == 1012: - self.__logger.info("WSS server is about to restart, clients need " \ - "to reconnect (server sent 20051). Reconnection attempt in progress...") + self.__logger.warning("WSS server is restarting: all " \ + "clients need to reconnect (server sent 20051).") if self.__timeout: asyncio.get_event_loop().call_later( @@ -177,10 +178,14 @@ class BfxWebSocketClient(Connection): _delay.reset() elif ((isinstance(error, InvalidStatusCode) and error.status_code == 408) or \ isinstance(error, gaierror)) and self.__reconnection: - self.__logger.warning( - f"_Reconnection attempt was unsuccessful (no.{self.__reconnection['attempts']}). " \ - f"Next reconnection attempt in {int(_delay.peek())}.0 seconds. (at the moment " \ - f"the client has been offline for {datetime.now() - self.__reconnection['timestamp']})") + #pylint: disable-next=logging-fstring-interpolation + self.__logger.warning("Reconnection attempt unsuccessful (no." \ + f"{self.__reconnection['attempts']}): next attempt in " \ + f"~{int(_delay.peek())}.0s.") + + #pylint: disable-next=logging-fstring-interpolation + self.__logger.info(f"The client has been offline for " \ + f"{datetime.now() - self.__reconnection['timestamp']}.") self.__reconnection["attempts"] += 1 else: @@ -196,9 +201,10 @@ class BfxWebSocketClient(Connection): async def __connect(self) -> None: async with websockets.client.connect(self._host) as websocket: if self.__reconnection: - self.__logger.info(f"_Reconnection attempt successful (no.{self.__reconnection['attempts']}): The " \ - f"client has been offline for a total of {datetime.now() - self.__reconnection['timestamp']} " \ - f"(connection lost on: {self.__reconnection['timestamp']:%d-%m-%Y at %H:%M:%S}).") + #pylint: disable-next=logging-fstring-interpolation + self.__logger.warning("Reconnection attempt successful (no." \ + f"{self.__reconnection['attempts']}): recovering " \ + "connection state...") self.__reconnection = None diff --git a/bfxapi/websocket/_handlers/auth_events_handler.py b/bfxapi/websocket/_handlers/auth_events_handler.py index d93364d..b6a0c91 100644 --- a/bfxapi/websocket/_handlers/auth_events_handler.py +++ b/bfxapi/websocket/_handlers/auth_events_handler.py @@ -37,7 +37,7 @@ class AuthEventsHandler: def handle(self, abbrevation: str, stream: Any) -> None: if abbrevation == "n": - return self.__notification(stream) + self.__notification(stream) for abbrevations, serializer in AuthEventsHandler.__SERIALIZERS.items(): if abbrevation in abbrevations: @@ -45,12 +45,11 @@ class AuthEventsHandler: if all(isinstance(sub_stream, list) for sub_stream in stream): data = [ serializer.parse(*sub_stream) for sub_stream in stream ] - else: data = serializer.parse(*stream) + else: + data = serializer.parse(*stream) self.__event_emitter.emit(event, data) - break - def __notification(self, stream: Any) -> None: event: str = "notification" diff --git a/bfxapi/websocket/_handlers/public_channels_handler.py b/bfxapi/websocket/_handlers/public_channels_handler.py index 9e46989..33b7af3 100644 --- a/bfxapi/websocket/_handlers/public_channels_handler.py +++ b/bfxapi/websocket/_handlers/public_channels_handler.py @@ -35,6 +35,7 @@ class PublicChannelsHandler: elif subscription["channel"] == "status": self.__status_channel_handler(cast(Status, subscription), stream) + #pylint: disable-next=inconsistent-return-statements def __ticker_channel_handler(self, subscription: Ticker, stream: List[Any]): if subscription["symbol"].startswith("t"): return self.__event_emitter.emit("t_ticker_update", subscription, \ @@ -44,6 +45,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("f_ticker_update", subscription, \ serializers.FundingCurrencyTicker.parse(*stream[0])) + #pylint: disable-next=inconsistent-return-statements def __trades_channel_handler(self, subscription: Trades, stream: List[Any]): if (event := stream[0]) and event in [ "te", "tu", "fte", "ftu" ]: events = { "te": "t_trade_execution", "tu": "t_trade_execution_update", \ @@ -67,6 +69,7 @@ class PublicChannelsHandler: [ serializers.FundingCurrencyTrade.parse(*sub_stream) \ for sub_stream in stream[0] ]) + #pylint: disable-next=inconsistent-return-statements def __book_channel_handler(self, subscription: Book, stream: List[Any]): if subscription["symbol"].startswith("t"): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): @@ -86,6 +89,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("f_book_update", subscription, \ serializers.FundingCurrencyBook.parse(*stream[0])) + #pylint: disable-next=inconsistent-return-statements def __raw_book_channel_handler(self, subscription: Book, stream: List[Any]): if subscription["symbol"].startswith("t"): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): @@ -105,6 +109,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("f_raw_book_update", subscription, \ serializers.FundingCurrencyRawBook.parse(*stream[0])) + #pylint: disable-next=inconsistent-return-statements def __candles_channel_handler(self, subscription: Candles, stream: List[Any]): if all(isinstance(sub_stream, list) for sub_stream in stream[0]): return self.__event_emitter.emit("candles_snapshot", subscription, \ @@ -114,6 +119,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("candles_update", subscription, \ serializers.Candle.parse(*stream[0])) + #pylint: disable-next=inconsistent-return-statements def __status_channel_handler(self, subscription: Status, stream: List[Any]): if subscription["key"].startswith("deriv:"): return self.__event_emitter.emit("derivatives_status_update", subscription, \ @@ -123,6 +129,7 @@ class PublicChannelsHandler: return self.__event_emitter.emit("liquidation_feed_update", subscription, \ serializers.Liquidation.parse(*stream[0][0])) + #pylint: disable-next=inconsistent-return-statements def __checksum_handler(self, subscription: Book, value: int): return self.__event_emitter.emit( \ "checksum", subscription, value & 0xFFFFFFFF) From 2734ff9e1a46ea973e2135c9132523a5f6205fc2 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 05:41:47 +0200 Subject: [PATCH 51/65] Drop modules bfxapi.enums, bfxapi.rest.enums and bfxapi.websocket.enums. --- bfxapi/__init__.py | 12 ++--- bfxapi/{client.py => _client.py} | 7 ++- bfxapi/{version.py => _version.py} | 0 bfxapi/enums.py | 50 ------------------- bfxapi/rest/endpoints/rest_auth_endpoints.py | 10 ++-- .../rest/endpoints/rest_merchant_endpoints.py | 8 ++- .../rest/endpoints/rest_public_endpoints.py | 26 +++++----- bfxapi/rest/enums.py | 47 ----------------- bfxapi/rest/middleware/middleware.py | 19 ++++--- bfxapi/urls.py | 5 -- .../websocket/_client/bfx_websocket_inputs.py | 8 +-- bfxapi/websocket/enums.py | 9 ---- setup.py | 8 +-- 13 files changed, 50 insertions(+), 159 deletions(-) rename bfxapi/{client.py => _client.py} (89%) rename bfxapi/{version.py => _version.py} (100%) delete mode 100644 bfxapi/enums.py delete mode 100644 bfxapi/rest/enums.py delete mode 100644 bfxapi/urls.py delete mode 100644 bfxapi/websocket/enums.py diff --git a/bfxapi/__init__.py b/bfxapi/__init__.py index b583248..9138036 100644 --- a/bfxapi/__init__.py +++ b/bfxapi/__init__.py @@ -1,6 +1,6 @@ -from .client import Client - -from .urls import REST_HOST, PUB_REST_HOST, \ - WSS_HOST, PUB_WSS_HOST - -from .version import __version__ +from ._client import \ + Client, \ + REST_HOST, \ + WSS_HOST, \ + PUB_REST_HOST, \ + PUB_WSS_HOST diff --git a/bfxapi/client.py b/bfxapi/_client.py similarity index 89% rename from bfxapi/client.py rename to bfxapi/_client.py index d45b925..baf81c6 100644 --- a/bfxapi/client.py +++ b/bfxapi/_client.py @@ -7,12 +7,17 @@ from bfxapi._exceptions import IncompleteCredentialError from bfxapi.rest import BfxRestInterface from bfxapi.websocket import BfxWebSocketClient -from bfxapi.urls import REST_HOST, WSS_HOST if TYPE_CHECKING: from bfxapi.websocket._client.bfx_websocket_client import \ _Credentials +REST_HOST = "https://api.bitfinex.com/v2" +WSS_HOST = "wss://api.bitfinex.com/ws/2" + +PUB_REST_HOST = "https://api-pub.bitfinex.com/v2" +PUB_WSS_HOST = "wss://api-pub.bitfinex.com/ws/2" + class Client: def __init__( self, diff --git a/bfxapi/version.py b/bfxapi/_version.py similarity index 100% rename from bfxapi/version.py rename to bfxapi/_version.py diff --git a/bfxapi/enums.py b/bfxapi/enums.py deleted file mode 100644 index 9b06bc2..0000000 --- a/bfxapi/enums.py +++ /dev/null @@ -1,50 +0,0 @@ -from enum import Enum - -class OrderType(str, Enum): - LIMIT = "LIMIT" - EXCHANGE_LIMIT = "EXCHANGE LIMIT" - MARKET = "MARKET" - EXCHANGE_MARKET = "EXCHANGE MARKET" - STOP = "STOP" - EXCHANGE_STOP = "EXCHANGE STOP" - STOP_LIMIT = "STOP LIMIT" - EXCHANGE_STOP_LIMIT = "EXCHANGE STOP LIMIT" - TRAILING_STOP = "TRAILING STOP" - EXCHANGE_TRAILING_STOP = "EXCHANGE TRAILING STOP" - FOK = "FOK" - EXCHANGE_FOK = "EXCHANGE FOK" - IOC = "IOC" - EXCHANGE_IOC = "EXCHANGE IOC" - -class FundingOfferType(str, Enum): - LIMIT = "LIMIT" - FRR_DELTA_FIX = "FRRDELTAFIX" - FRR_DELTA_VAR = "FRRDELTAVAR" - -class Flag(int, Enum): - HIDDEN = 64 - CLOSE = 512 - REDUCE_ONLY = 1024 - POST_ONLY = 4096 - OCO = 16384 - NO_VAR_RATES = 524288 - -class Error(int, Enum): - ERR_UNK = 10000 - ERR_GENERIC = 10001 - ERR_CONCURRENCY = 10008 - ERR_PARAMS = 10020 - ERR_CONF_FAIL = 10050 - ERR_AUTH_FAIL = 10100 - ERR_AUTH_PAYLOAD = 10111 - ERR_AUTH_SIG = 10112 - ERR_AUTH_HMAC = 10113 - ERR_AUTH_NONCE = 10114 - ERR_UNAUTH_FAIL = 10200 - ERR_SUB_FAIL = 10300 - ERR_SUB_MULTI = 10301 - ERR_SUB_UNK = 10302 - ERR_SUB_LIMIT = 10305 - ERR_UNSUB_FAIL = 10400 - ERR_UNSUB_NOT = 10401 - ERR_READY = 11000 diff --git a/bfxapi/rest/endpoints/rest_auth_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py index 3dc0885..658e182 100644 --- a/bfxapi/rest/endpoints/rest_auth_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -4,8 +4,6 @@ from decimal import Decimal from ..middleware import Middleware -from ..enums import Sort, OrderType, FundingOfferType - from ...types import Notification, \ UserInfo, LoginHistory, BalanceAvailable, \ Order, Position, Trade, \ @@ -63,7 +61,7 @@ class RestAuthEndpoints(Middleware): for sub_data in self._post(endpoint, body={ "id": ids }) ] def submit_order(self, - type: OrderType, + type: str, symbol: str, amount: Union[Decimal, float, str], *, @@ -163,7 +161,7 @@ class RestAuthEndpoints(Middleware): def get_trades_history(self, *, symbol: Optional[str] = None, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[Trade]: @@ -285,7 +283,7 @@ class RestAuthEndpoints(Middleware): #pylint: disable-next=too-many-arguments def submit_funding_offer(self, - type: FundingOfferType, + type: str, symbol: str, amount: Union[Decimal, float, str], rate: Union[Decimal, float, str], @@ -397,7 +395,7 @@ class RestAuthEndpoints(Middleware): def get_funding_trades_history(self, *, symbol: Optional[str] = None, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[FundingTrade]: diff --git a/bfxapi/rest/endpoints/rest_merchant_endpoints.py b/bfxapi/rest/endpoints/rest_merchant_endpoints.py index d201a10..d34028b 100644 --- a/bfxapi/rest/endpoints/rest_merchant_endpoints.py +++ b/bfxapi/rest/endpoints/rest_merchant_endpoints.py @@ -7,8 +7,6 @@ from decimal import Decimal from bfxapi.rest.middleware import Middleware -from bfxapi.rest.enums import MerchantSettingsKey - from bfxapi.types import \ InvoiceSubmission, \ InvoicePage, \ @@ -140,16 +138,16 @@ class RestMerchantEndpoints(Middleware): body={ "baseCcy": base_ccy, "convertCcy": convert_ccy })) def set_merchant_settings(self, - key: MerchantSettingsKey, + key: str, val: Any) -> bool: return bool(self._post("auth/w/ext/pay/settings/set", \ body={ "key": key, "val": val })) - def get_merchant_settings(self, key: MerchantSettingsKey) -> Any: + def get_merchant_settings(self, key: str) -> Any: return self._post("auth/r/ext/pay/settings/get", body={ "key": key }) #pylint: disable-next=dangerous-default-value - def list_merchant_settings(self, keys: List[MerchantSettingsKey] = []) -> Dict[MerchantSettingsKey, Any]: + def list_merchant_settings(self, keys: List[str] = []) -> Dict[str, Any]: return self._post("auth/r/ext/pay/settings/list", body={ "keys": keys }) def get_deposits(self, diff --git a/bfxapi/rest/endpoints/rest_public_endpoints.py b/bfxapi/rest/endpoints/rest_public_endpoints.py index 4401057..5549730 100644 --- a/bfxapi/rest/endpoints/rest_public_endpoints.py +++ b/bfxapi/rest/endpoints/rest_public_endpoints.py @@ -4,8 +4,6 @@ from decimal import Decimal from ..middleware import Middleware -from ..enums import Config, Sort - from ...types import \ PlatformStatus, TradingPairTicker, FundingCurrencyTicker, \ TickersHistory, TradingPairTrade, FundingCurrencyTrade, \ @@ -19,7 +17,7 @@ from ...types import serializers #pylint: disable-next=too-many-public-methods class RestPublicEndpoints(Middleware): - def conf(self, config: Config) -> Any: + def conf(self, config: str) -> Any: return self._get(f"conf/{config}")[0] def get_platform_status(self) -> PlatformStatus: @@ -84,7 +82,7 @@ class RestPublicEndpoints(Middleware): limit: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, - sort: Optional[Sort] = None) -> List[TradingPairTrade]: + sort: Optional[int] = None) -> List[TradingPairTrade]: params = { "limit": limit, "start": start, "end": end, "sort": sort } data = self._get(f"trades/{pair}/hist", params=params) return [ serializers.TradingPairTrade.parse(*sub_data) for sub_data in data ] @@ -95,7 +93,7 @@ class RestPublicEndpoints(Middleware): limit: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, - sort: Optional[Sort] = None) -> List[FundingCurrencyTrade]: + sort: Optional[int] = None) -> List[FundingCurrencyTrade]: params = { "limit": limit, "start": start, "end": end, "sort": sort } data = self._get(f"trades/{currency}/hist", params=params) return [ serializers.FundingCurrencyTrade.parse(*sub_data) for sub_data in data ] @@ -133,7 +131,7 @@ class RestPublicEndpoints(Middleware): def get_stats_hist(self, resource: str, *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[Statistic]: @@ -144,7 +142,7 @@ class RestPublicEndpoints(Middleware): def get_stats_last(self, resource: str, *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> Statistic: @@ -156,7 +154,7 @@ class RestPublicEndpoints(Middleware): symbol: str, tf: str = "1m", *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[Candle]: @@ -168,7 +166,7 @@ class RestPublicEndpoints(Middleware): symbol: str, tf: str = "1m", *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> Candle: @@ -192,7 +190,7 @@ class RestPublicEndpoints(Middleware): def get_derivatives_status_history(self, key: str, *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[DerivativesStatus]: @@ -202,7 +200,7 @@ class RestPublicEndpoints(Middleware): def get_liquidations(self, *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[Liquidation]: @@ -214,7 +212,7 @@ class RestPublicEndpoints(Middleware): symbol: str, tf: str = "1m", *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[Candle]: @@ -225,7 +223,7 @@ class RestPublicEndpoints(Middleware): def get_leaderboards_hist(self, resource: str, *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> List[Leaderboard]: @@ -236,7 +234,7 @@ class RestPublicEndpoints(Middleware): def get_leaderboards_last(self, resource: str, *, - sort: Optional[Sort] = None, + sort: Optional[int] = None, start: Optional[str] = None, end: Optional[str] = None, limit: Optional[int] = None) -> Leaderboard: diff --git a/bfxapi/rest/enums.py b/bfxapi/rest/enums.py deleted file mode 100644 index 17b3753..0000000 --- a/bfxapi/rest/enums.py +++ /dev/null @@ -1,47 +0,0 @@ -#pylint: disable-next=wildcard-import,unused-wildcard-import -from ..enums import * - -class Config(str, Enum): - MAP_CURRENCY_SYM = "pub:map:currency:sym" - MAP_CURRENCY_LABEL = "pub:map:currency:label" - MAP_CURRENCY_UNIT = "pub:map:currency:unit" - MAP_CURRENCY_UNDL = "pub:map:currency:undl" - MAP_CURRENCY_POOL = "pub:map:currency:pool" - MAP_CURRENCY_EXPLORER = "pub:map:currency:explorer" - MAP_CURRENCY_TX_FEE = "pub:map:currency:tx:fee" - MAP_TX_METHOD = "pub:map:tx:method" - - LIST_PAIR_EXCHANGE = "pub:list:pair:exchange" - LIST_PAIR_MARGIN = "pub:list:pair:margin" - LIST_PAIR_FUTURES = "pub:list:pair:futures" - LIST_PAIR_SECURITIES = "pub:list:pair:securities" - LIST_CURRENCY = "pub:list:currency" - LIST_COMPETITIONS = "pub:list:competitions" - - INFO_PAIR = "pub:info:pair" - INFO_PAIR_FUTURES = "pub:info:pair:futures" - INFO_TX_STATUS = "pub:info:tx:status" - - SPEC_MARGIN = "pub:spec:margin" - FEES = "pub:fees" - -class Precision(str, Enum): - P0 = "P0" - P1 = "P1" - P2 = "P2" - P3 = "P3" - P4 = "P4" - -class Sort(int, Enum): - ASCENDING = +1 - DESCENDING = -1 - -class MerchantSettingsKey(str, Enum): - PREFERRED_FIAT = "bfx_pay_preferred_fiat" - RECOMMEND_STORE = "bfx_pay_recommend_store" - NOTIFY_PAYMENT_COMPLETED = "bfx_pay_notify_payment_completed" - NOTIFY_PAYMENT_COMPLETED_EMAIL = "bfx_pay_notify_payment_completed_email" - NOTIFY_AUTOCONVERT_EXECUTED = "bfx_pay_notify_autoconvert_executed" - DUST_BALANCE_UI = "bfx_pay_dust_balance_ui" - MERCHANT_CUSTOMER_SUPPORT_URL = "bfx_pay_merchant_customer_support_url" - MERCHANT_UNDERPAID_THRESHOLD = "bfx_pay_merchant_underpaid_threshold" diff --git a/bfxapi/rest/middleware/middleware.py b/bfxapi/rest/middleware/middleware.py index cf434e5..4d29418 100644 --- a/bfxapi/rest/middleware/middleware.py +++ b/bfxapi/rest/middleware/middleware.py @@ -1,10 +1,11 @@ from typing import TYPE_CHECKING, Optional, Any +from enum import Enum + from http import HTTPStatus import time, hmac, hashlib, json, requests -from ..enums import Error from ..exceptions import ResourceNotFound, RequestParametersError, InvalidCredentialError, UnknownGenericError from ..._utils.json_encoder import JSONEncoder from ..._utils.json_decoder import JSONDecoder @@ -12,6 +13,12 @@ from ..._utils.json_decoder import JSONDecoder if TYPE_CHECKING: from requests.sessions import _Params +class _Error(Enum): + ERR_UNK = 10000 + ERR_GENERIC = 10001 + ERR_PARAMS = 10020 + ERR_AUTH_FAIL = 10100 + class Middleware: TIMEOUT = 30 @@ -53,11 +60,11 @@ class Middleware: data = response.json(cls=JSONDecoder) if len(data) and data[0] == "error": - if data[1] == Error.ERR_PARAMS: + if data[1] == _Error.ERR_PARAMS: raise RequestParametersError("The request was rejected with the " \ f"following parameter error: <{data[2]}>") - if data[1] is None or data[1] == Error.ERR_UNK or data[1] == Error.ERR_GENERIC: + if data[1] is None or data[1] == _Error.ERR_UNK or data[1] == _Error.ERR_GENERIC: raise UnknownGenericError("The server replied to the request with " \ f"a generic error with message: <{data[2]}>.") @@ -86,14 +93,14 @@ class Middleware: data = response.json(cls=JSONDecoder) if isinstance(data, list) and len(data) and data[0] == "error": - if data[1] == Error.ERR_PARAMS: + if data[1] == _Error.ERR_PARAMS: raise RequestParametersError("The request was rejected with the " \ f"following parameter error: <{data[2]}>") - if data[1] == Error.ERR_AUTH_FAIL: + if data[1] == _Error.ERR_AUTH_FAIL: raise InvalidCredentialError("Cannot authenticate with given API-KEY and API-SECRET.") - if data[1] is None or data[1] == Error.ERR_UNK or data[1] == Error.ERR_GENERIC: + if data[1] is None or data[1] == _Error.ERR_UNK or data[1] == _Error.ERR_GENERIC: raise UnknownGenericError("The server replied to the request with " \ f"a generic error with message: <{data[2]}>.") diff --git a/bfxapi/urls.py b/bfxapi/urls.py deleted file mode 100644 index 556e4d9..0000000 --- a/bfxapi/urls.py +++ /dev/null @@ -1,5 +0,0 @@ -REST_HOST = "https://api.bitfinex.com/v2" -PUB_REST_HOST = "https://api-pub.bitfinex.com/v2" - -WSS_HOST = "wss://api.bitfinex.com/ws/2" -PUB_WSS_HOST = "wss://api-pub.bitfinex.com/ws/2" diff --git a/bfxapi/websocket/_client/bfx_websocket_inputs.py b/bfxapi/websocket/_client/bfx_websocket_inputs.py index f14d32d..1753f3a 100644 --- a/bfxapi/websocket/_client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/_client/bfx_websocket_inputs.py @@ -5,10 +5,6 @@ from typing import \ from decimal import Decimal -from bfxapi.enums import \ - OrderType, \ - FundingOfferType - _Handler = Callable[[str, Any], Awaitable[None]] class BfxWebSocketInputs: @@ -16,7 +12,7 @@ class BfxWebSocketInputs: self.__handle_websocket_input = handle_websocket_input async def submit_order(self, - type: OrderType, + type: str, symbol: str, amount: Union[str, float, Decimal], price: Union[str, float, Decimal], @@ -79,7 +75,7 @@ class BfxWebSocketInputs: #pylint: disable-next=too-many-arguments async def submit_funding_offer(self, - type: FundingOfferType, + type: str, symbol: str, amount: Union[str, float, Decimal], rate: Union[str, float, Decimal], diff --git a/bfxapi/websocket/enums.py b/bfxapi/websocket/enums.py deleted file mode 100644 index 8fe6028..0000000 --- a/bfxapi/websocket/enums.py +++ /dev/null @@ -1,9 +0,0 @@ -#pylint: disable-next=wildcard-import,unused-wildcard-import -from bfxapi.enums import * - -class Channel(str, Enum): - TICKER = "ticker" - TRADES = "trades" - BOOK = "book" - CANDLES = "candles" - STATUS = "status" diff --git a/setup.py b/setup.py index 9ca14a8..f79a84f 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,12 @@ from distutils.core import setup -version = {} -with open("bfxapi/version.py", encoding="utf-8") as fp: - exec(fp.read(), version) #pylint: disable=exec-used +_version = {} +with open("bfxapi/_version.py", encoding="utf-8") as fp: + exec(fp.read(), _version) #pylint: disable=exec-used setup( name="bitfinex-api-py", - version=version["__version__"], + version=_version["__version__"], description="Official Bitfinex Python API", long_description="A Python reference implementation of the Bitfinex API for both REST and websocket interaction", long_description_content_type="text/markdown", From c02d6d7bf88947bab266383a9d5b09a13649919c Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 05:57:10 +0200 Subject: [PATCH 52/65] Fix bug in module bfxapi.websocket._event_emitter. --- .../websocket/_client/bfx_websocket_client.py | 2 +- .../_event_emitter/bfx_event_emitter.py | 49 +++++++++++-------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index a8831c0..eb7d9ab 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -335,4 +335,4 @@ class BfxWebSocketClient(Connection): [ 0, event, None, data], cls=JSONEncoder)) def on(self, event, f = None): - return self.__event_emitter.on(event, f=f) + return self.__event_emitter.on(event, f) diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py index 31ee983..27f00d0 100644 --- a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -1,6 +1,7 @@ from typing import \ - Callable, List, Dict, \ - Optional, Any + TypeVar, Callable, List, \ + Dict, Union, Optional, \ + Any from collections import defaultdict from asyncio import AbstractEventLoop @@ -8,6 +9,8 @@ from pyee.asyncio import AsyncIOEventEmitter from bfxapi.websocket.exceptions import UnknownEventError +_Handler = TypeVar("_Handler", bound=Callable[..., None]) + _ONCE_PER_CONNECTION = [ "open", "authenticated", "order_snapshot", "position_snapshot", "funding_offer_snapshot", "funding_credit_snapshot", @@ -21,19 +24,19 @@ _ONCE_PER_SUBSCRIPTION = [ ] _COMMON = [ - "error", "disconnected", "t_ticker_update", - "f_ticker_update", "t_trade_execution", "t_trade_execution_update", - "f_trade_execution", "f_trade_execution_update", "t_book_update", - "f_book_update", "t_raw_book_update", "f_raw_book_update", - "candles_update", "derivatives_status_update", "liquidation_feed_update", - "order_new", "order_update", "order_cancel", - "position_new", "position_update", "position_close", - "funding_offer_new", "funding_offer_update", "funding_offer_cancel", - "funding_credit_new", "funding_credit_update", "funding_credit_close", - "funding_loan_new", "funding_loan_update", "funding_loan_close", - "trade_execution", "trade_execution_update", "wallet_update", - "notification", "on-req-notification", "ou-req-notification", - "oc-req-notification", "fon-req-notification", "foc-req-notification" + "disconnected", "t_ticker_update", "f_ticker_update", + "t_trade_execution", "t_trade_execution_update", "f_trade_execution", + "f_trade_execution_update", "t_book_update", "f_book_update", + "t_raw_book_update", "f_raw_book_update", "candles_update", + "derivatives_status_update", "liquidation_feed_update", "order_new", + "order_update", "order_cancel", "position_new", + "position_update", "position_close", "funding_offer_new", + "funding_offer_update", "funding_offer_cancel", "funding_credit_new", + "funding_credit_update", "funding_credit_close", "funding_loan_new", + "funding_loan_update", "funding_loan_close", "trade_execution", + "trade_execution_update", "wallet_update", "notification", + "on-req-notification", "ou-req-notification", "oc-req-notification", + "fon-req-notification", "foc-req-notification" ] class BfxEventEmitter(AsyncIOEventEmitter): @@ -49,10 +52,12 @@ class BfxEventEmitter(AsyncIOEventEmitter): self._subscriptions: Dict[str, List[str]] = \ defaultdict(lambda: [ ]) - def emit(self, - event: str, - *args: Any, - **kwargs: Any) -> bool: + def emit( + self, + event: str, + *args: Any, + **kwargs: Any + ) -> bool: if event in _ONCE_PER_CONNECTION: if event in self._connection: return self._has_listeners(event) @@ -69,12 +74,14 @@ class BfxEventEmitter(AsyncIOEventEmitter): return super().emit(event, *args, **kwargs) - def _add_event_handler(self, event: str, k: Callable, v: Callable): + def on( + self, event: str, f: Optional[_Handler] = None + ) -> Union[_Handler, Callable[[_Handler], _Handler]]: if event not in BfxEventEmitter._EVENTS: raise UnknownEventError(f"Can't register to unknown event: <{event}> " + \ "(to get a full list of available events see https://docs.bitfinex.com/).") - super()._add_event_handler(event, k, v) + return super().on(event, f) def _has_listeners(self, event: str) -> bool: with self._lock: From b082891c418c1db33b9b89b3e462487fd80c10fa Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 06:41:42 +0200 Subject: [PATCH 53/65] Remove useless and redundant docstrings from custom exceptions. --- bfxapi/_client.py | 3 +- bfxapi/_exceptions.py | 14 ----- bfxapi/exceptions.py | 10 ++++ bfxapi/rest/exceptions.py | 17 ++---- bfxapi/rest/middleware/middleware.py | 8 +-- .../websocket/_client/bfx_websocket_client.py | 4 +- bfxapi/websocket/exceptions.py | 54 ++++++------------- 7 files changed, 40 insertions(+), 70 deletions(-) delete mode 100644 bfxapi/_exceptions.py create mode 100644 bfxapi/exceptions.py diff --git a/bfxapi/_client.py b/bfxapi/_client.py index baf81c6..2a7d8f0 100644 --- a/bfxapi/_client.py +++ b/bfxapi/_client.py @@ -3,10 +3,9 @@ from typing import \ from bfxapi._utils.logging import ColorLogger -from bfxapi._exceptions import IncompleteCredentialError - from bfxapi.rest import BfxRestInterface from bfxapi.websocket import BfxWebSocketClient +from bfxapi.exceptions import IncompleteCredentialError if TYPE_CHECKING: from bfxapi.websocket._client.bfx_websocket_client import \ diff --git a/bfxapi/_exceptions.py b/bfxapi/_exceptions.py deleted file mode 100644 index e408bd6..0000000 --- a/bfxapi/_exceptions.py +++ /dev/null @@ -1,14 +0,0 @@ -class BfxBaseException(Exception): - """ - Base class for every custom exception in bfxapi/rest/exceptions.py and bfxapi/websocket/exceptions.py. - """ - -class IncompleteCredentialError(BfxBaseException): - """ - This error indicates an incomplete credential object (missing api-key or api-secret). - """ - -class InvalidCredentialError(BfxBaseException): - """ - This error indicates that the user has provided incorrect credentials (API-KEY and API-SECRET) for authentication. - """ diff --git a/bfxapi/exceptions.py b/bfxapi/exceptions.py new file mode 100644 index 0000000..663752a --- /dev/null +++ b/bfxapi/exceptions.py @@ -0,0 +1,10 @@ +class BfxBaseException(Exception): + """ + Base class for every custom exception thrown by bitfinex-api-py. + """ + +class IncompleteCredentialError(BfxBaseException): + pass + +class InvalidCredentialError(BfxBaseException): + pass diff --git a/bfxapi/rest/exceptions.py b/bfxapi/rest/exceptions.py index ab7001c..2906fcd 100644 --- a/bfxapi/rest/exceptions.py +++ b/bfxapi/rest/exceptions.py @@ -1,17 +1,10 @@ -# pylint: disable-next=wildcard-import,unused-wildcard-import -from bfxapi._exceptions import * +from bfxapi.exceptions import BfxBaseException -class ResourceNotFound(BfxBaseException): - """ - This error indicates a failed HTTP request to a non-existent resource. - """ +class NotFoundError(BfxBaseException): + pass class RequestParametersError(BfxBaseException): - """ - This error indicates that there are some invalid parameters sent along with an HTTP request. - """ + pass class UnknownGenericError(BfxBaseException): - """ - This error indicates an undefined problem processing an HTTP request sent to the APIs. - """ + pass diff --git a/bfxapi/rest/middleware/middleware.py b/bfxapi/rest/middleware/middleware.py index 4d29418..f279f30 100644 --- a/bfxapi/rest/middleware/middleware.py +++ b/bfxapi/rest/middleware/middleware.py @@ -6,7 +6,9 @@ from http import HTTPStatus import time, hmac, hashlib, json, requests -from ..exceptions import ResourceNotFound, RequestParametersError, InvalidCredentialError, UnknownGenericError +from ..exceptions import NotFoundError, RequestParametersError, UnknownGenericError + +from ...exceptions import InvalidCredentialError from ..._utils.json_encoder import JSONEncoder from ..._utils.json_decoder import JSONDecoder @@ -55,7 +57,7 @@ class Middleware: ) if response.status_code == HTTPStatus.NOT_FOUND: - raise ResourceNotFound(f"No resources found at endpoint <{endpoint}>.") + raise NotFoundError(f"No resources found at endpoint <{endpoint}>.") data = response.json(cls=JSONDecoder) @@ -88,7 +90,7 @@ class Middleware: ) if response.status_code == HTTPStatus.NOT_FOUND: - raise ResourceNotFound(f"No resources found at endpoint <{endpoint}>.") + raise NotFoundError(f"No resources found at endpoint <{endpoint}>.") data = response.json(cls=JSONDecoder) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index eb7d9ab..df6e8d5 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -24,8 +24,10 @@ from bfxapi.websocket._connection import Connection from bfxapi.websocket._handlers import AuthEventsHandler from bfxapi.websocket._event_emitter import BfxEventEmitter +from bfxapi.exceptions import \ + InvalidCredentialError + from bfxapi.websocket.exceptions import \ - InvalidCredentialError, \ ReconnectionTimeoutError, \ VersionMismatchError, \ UnknownChannelError, \ diff --git a/bfxapi/websocket/exceptions.py b/bfxapi/websocket/exceptions.py index e662cbf..fc9aae8 100644 --- a/bfxapi/websocket/exceptions.py +++ b/bfxapi/websocket/exceptions.py @@ -1,47 +1,25 @@ -# pylint: disable-next=wildcard-import,unused-wildcard-import -from bfxapi._exceptions import * +from bfxapi.exceptions import BfxBaseException class ConnectionNotOpen(BfxBaseException): - """ - This error indicates an attempt to communicate via websocket before starting the connection with the servers. - """ - -class FullBucketError(BfxBaseException): - """ - Thrown when a user attempts a subscription but all buckets are full. - """ - -class ReconnectionTimeoutError(BfxBaseException): - """ - This error indicates that the connection has been offline for too long without being able to reconnect. - """ + pass class ActionRequiresAuthentication(BfxBaseException): - """ - This error indicates an attempt to access a protected resource without logging in first. - """ + pass -class UnknownChannelError(BfxBaseException): - """ - Thrown when a user attempts to subscribe to an unknown channel. - """ - -class UnknownSubscriptionError(BfxBaseException): - """ - Thrown when a user attempts to reference an unknown subscription. - """ - -class UnknownEventError(BfxBaseException): - """ - Thrown when a user attempts to add a listener for an unknown event. - """ +class ReconnectionTimeoutError(BfxBaseException): + pass class VersionMismatchError(BfxBaseException): - """ - This error indicates a mismatch between the client version and the server WSS version. - """ + pass class SubIdError(BfxBaseException): - """ - Thrown when a user attempts to open more than one subscription using the same sub_id. - """ + pass + +class UnknownChannelError(BfxBaseException): + pass + +class UnknownEventError(BfxBaseException): + pass + +class UnknownSubscriptionError(BfxBaseException): + pass From 77494de9efeb4858b28164fec96cbac5359eb904 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 06:56:09 +0200 Subject: [PATCH 54/65] Remove old test suite in module bfxapi.tests. --- .github/workflows/bitfinex-api-py-ci.yml | 2 - .travis.yml | 11 ----- README.md | 16 ------- bfxapi/tests/__init__.py | 15 ------- bfxapi/tests/test_types_labeler.py | 56 ------------------------ bfxapi/tests/test_types_notification.py | 29 ------------ bfxapi/tests/test_types_serializers.py | 17 ------- 7 files changed, 146 deletions(-) delete mode 100644 .travis.yml delete mode 100644 bfxapi/tests/__init__.py delete mode 100644 bfxapi/tests/test_types_labeler.py delete mode 100644 bfxapi/tests/test_types_notification.py delete mode 100644 bfxapi/tests/test_types_serializers.py diff --git a/.github/workflows/bitfinex-api-py-ci.yml b/.github/workflows/bitfinex-api-py-ci.yml index 295ce9f..9c2cbb7 100644 --- a/.github/workflows/bitfinex-api-py-ci.yml +++ b/.github/workflows/bitfinex-api-py-ci.yml @@ -27,5 +27,3 @@ jobs: run: python -m pylint bfxapi - name: Run mypy to check the correctness of type hinting (and fail if any error or warning is found) run: python -m mypy bfxapi - - name: Execute project's unit tests (unittest) - run: python -m unittest bfxapi.tests diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c9210f6..0000000 --- a/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: python -python: - - "3.8.0" -before_install: - - python -m pip install --upgrade pip -install: - - pip install -r dev-requirements.txt -script: - - python -m pylint bfxapi - - python -m mypy bfxapi - - python -m unittest bfxapi.tests diff --git a/README.md b/README.md index 38caf3d..a564c54 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,6 @@ Contributors must uphold the [Contributor Covenant code of conduct](https://gith * [Cloning the repository](#cloning-the-repository) * [Installing the dependencies](#installing-the-dependencies) 2. [Before opening a PR](#before-opening-a-pr) - * [Running the unit tests](#running-the-unit-tests) 3. [License](#license) ## Installation and setup @@ -349,24 +348,9 @@ Wheter you're submitting a bug fix, a new feature or a documentation change, you All PRs must follow this [PULL_REQUEST_TEMPLATE](https://github.com/bitfinexcom/bitfinex-api-py/blob/v3-beta/.github/PULL_REQUEST_TEMPLATE.md) and include an exhaustive description. Before opening a pull request, you should also make sure that: -- [ ] all unit tests pass (see [Running the unit tests](#running-the-unit-tests)). - [ ] [`pylint`](https://github.com/pylint-dev/pylint) returns a score of 10.00/10.00 when run against your code. - [ ] [`mypy`](https://github.com/python/mypy) doesn't throw any error code when run on the project (excluding notes). -### Running the unit tests - -`bitfinex-api-py` comes with a set of unit tests (written using the [`unittest`](https://docs.python.org/3.8/library/unittest.html) unit testing framework). \ -Contributors must ensure that each unit test passes before opening a pull request. \ -You can run all project's unit tests by calling `unittest` on `bfxapi.tests`: -```console -python3 -m unittest -v bfxapi.tests -``` - -A single unit test can be run as follows: -```console -python3 -m unittest -v bfxapi.tests.test_notification -``` - ## License ``` diff --git a/bfxapi/tests/__init__.py b/bfxapi/tests/__init__.py deleted file mode 100644 index e7a6f4e..0000000 --- a/bfxapi/tests/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from .test_types_labeler import TestTypesLabeler -from .test_types_notification import TestTypesNotification -from .test_types_serializers import TestTypesSerializers - -def suite(): - return unittest.TestSuite([ - unittest.makeSuite(TestTypesLabeler), - unittest.makeSuite(TestTypesNotification), - unittest.makeSuite(TestTypesSerializers), - ]) - -if __name__ == "__main__": - unittest.TextTestRunner().run(suite()) diff --git a/bfxapi/tests/test_types_labeler.py b/bfxapi/tests/test_types_labeler.py deleted file mode 100644 index 639736b..0000000 --- a/bfxapi/tests/test_types_labeler.py +++ /dev/null @@ -1,56 +0,0 @@ -import unittest - -from typing import Optional - -from dataclasses import dataclass - -from ..types.labeler import _Type, generate_labeler_serializer, generate_recursive_serializer - -class TestTypesLabeler(unittest.TestCase): - def test_generate_labeler_serializer(self): - @dataclass - class Test(_Type): - A: Optional[int] - B: float - C: str - - labels = [ "A", "_PLACEHOLDER", "B", "_PLACEHOLDER", "C" ] - - serializer = generate_labeler_serializer("Test", Test, labels) - - self.assertEqual(serializer.parse(5, None, 65.0, None, "X"), Test(5, 65.0, "X"), - msg="_Serializer should produce the right result.") - - self.assertListEqual(serializer.get_labels(), [ "A", "B", "C" ], - msg="_Serializer::get_labels() should return the right list of labels.") - - with self.assertRaises(AssertionError, - msg="_Serializer should raise an AssertionError if given " \ - "fewer arguments than the serializer labels."): - serializer.parse(5, 65.0, "X") - - def test_generate_recursive_serializer(self): - @dataclass - class Outer(_Type): - A: int - B: float - C: "Middle" - - @dataclass - class Middle(_Type): - D: str - E: "Inner" - - @dataclass - class Inner(_Type): - F: bool - - inner = generate_labeler_serializer("Inner", Inner, ["F"]) - middle = generate_recursive_serializer("Middle", Middle, ["D", "E"], serializers={ "E": inner }) - outer = generate_recursive_serializer("Outer", Outer, ["A", "B", "C"], serializers={ "C": middle }) - - self.assertEqual(outer.parse(10, 45.5, [ "Y", [ True ] ]), Outer(10, 45.5, Middle("Y", Inner(True))), - msg="_RecursiveSerializer should produce the right result.") - -if __name__ == "__main__": - unittest.main() diff --git a/bfxapi/tests/test_types_notification.py b/bfxapi/tests/test_types_notification.py deleted file mode 100644 index 007f263..0000000 --- a/bfxapi/tests/test_types_notification.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from dataclasses import dataclass -from ..types.labeler import generate_labeler_serializer -from ..types.notification import _Type, _Notification, Notification - -class TestTypesNotification(unittest.TestCase): - def test_types_notification(self): - @dataclass - class Test(_Type): - A: int - B: float - C: str - - test = generate_labeler_serializer("Test", Test, - [ "A", "_PLACEHOLDER", "B", "_PLACEHOLDER", "C" ]) - - notification = _Notification[Test](test) - - actual = notification.parse(*[ 1675787861506, "test", None, None, [ 5, None, 65.0, None, "X" ], \ - 0, "SUCCESS", "This is just a test notification." ]) - - expected = Notification[Test](1675787861506, "test", None, Test(5, 65.0, "X"), - 0, "SUCCESS", "This is just a test notification.") - - self.assertEqual(actual, expected, msg="_Notification should produce the right notification.") - -if __name__ == "__main__": - unittest.main() diff --git a/bfxapi/tests/test_types_serializers.py b/bfxapi/tests/test_types_serializers.py deleted file mode 100644 index b5b2695..0000000 --- a/bfxapi/tests/test_types_serializers.py +++ /dev/null @@ -1,17 +0,0 @@ -import unittest -from ..types import serializers -from ..types.labeler import _Type - -class TestTypesSerializers(unittest.TestCase): - def test_types_serializers(self): - for serializer in map(serializers.__dict__.get, serializers.__serializers__): - self.assertTrue(issubclass(serializer.klass, _Type), - f"_Serializer <{serializer.name}>: .klass field must be a subclass " \ - f"of _Type (got {serializer.klass}).") - - self.assertListEqual(serializer.get_labels(), list(serializer.klass.__annotations__), - f"_Serializer <{serializer.name}> and _Type <{serializer.klass.__name__}> " \ - "must have matching labels and fields.") - -if __name__ == "__main__": - unittest.main() From 36c48c3b3fa7aae5451f57b6ddd4265b7ba6efa8 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 07:00:06 +0200 Subject: [PATCH 55/65] Apply small changes to .github/ISSUE_TEMPLATE.md and .github/PULL_REQUEST_TEMPLATE.md. --- .github/ISSUE_TEMPLATE.md | 7 +------ .github/PULL_REQUEST_TEMPLATE.md | 3 --- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d1f5f1d..c8b9498 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -26,10 +26,5 @@ A possible solution could be... ### Python version - + Python 3.10.6 x64 - -### Mypy version - - -mypy 0.991 (compiled: yes) \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f08ca31..05f83fa 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -25,8 +25,5 @@ PR fixes the following issue: - [ ] I have commented my code, particularly in hard-to-understand areas; - [ ] I have made corresponding changes to the documentation; - [ ] My changes generate no new warnings; -- [ ] I have added tests that prove my fix is effective or that my feature works; -- [ ] New and existing unit tests pass locally with my changes; - [ ] Mypy returns no errors or warnings when run on the root package; - [ ] Pylint returns a score of 10.00/10.00 when run on the root package; -- [ ] I have updated the library version and updated the CHANGELOG; \ No newline at end of file From 1e7a4d5371c94910a988bc17e3e2fd9152ef9ceb Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 07:21:36 +0200 Subject: [PATCH 56/65] Upgrade dependencies in requirements.txt and dev-requirements.txt. --- dev-requirements.txt | Bin 600 -> 216 bytes requirements.txt | Bin 300 -> 140 bytes setup.py | 5 +++-- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index c7daa7862d8b787e838ecbbee8fcad38190de71d..0cd0c125ab44449379d02fdf2c09ca6c6aa0f460 100644 GIT binary patch delta 15 Wcmcb?a)WV#1EcB0-tx(R7!?39P6h!0 literal 600 zcmZXR>rTQz5QXQriSOVGl(6NZ{NQ1r)TG+gQkIB5yz-q{w$T_eg>o)4bLQu}ve6!> z){b_z&VNPT-YnV4W;`#{z1?|5`?P`D@(G{|YW@eQ&`KeP6MpahUHJ2fpFpDZFof0q z{|rxcO5UYKuj%q+decOOUE33y;8U#rsm|znci%H(>0a9!O>HZlOV_B|&Yef*>FhB1 z2WREo8nB)k;YLb++GI(s-RFQ-{R*O7GScBLWpYfN!);sj?s5BK+X~L!Uf8vPS&^~k zx%^6cgR&&uz}mrYr+$wUI=E-DXYU%>NH@*7X)0YU`{<6!UzaYa22UtwQ!exhCo{Bj QPCW1S;fV}4RL&)}D~Os{O#lD@ diff --git a/requirements.txt b/requirements.txt index b2a3b7683962381e55063888496c3fa4478452d1..d5d501a945720c529673cff4427a7c23c9a2cac1 100644 GIT binary patch delta 46 ucmZ3()WazH|6c(^B||C@+5({^gC2tcgWhCTMnz6T2%EupVz=hR7b*Y^RSMMr literal 300 zcmX|+Ne+TQ6hv!n;!z|BI4wMj0YoJtAVVAuufF~uA-|!ktEwO0PpV9fI^{Ysd!9f( z6Ufuj){W+xDb+kUyOQ)|8OXQu8@{pqAZ zy{l(!xuqR>JUF&fXGglGxzt$)oKKt?yB4jox8sQLudLf~^sp92nkckZ#~5gBtpnAA RmV}myf4%bkzkfOM{QzW|E-(N9 diff --git a/setup.py b/setup.py index f79a84f..9951f6e 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,9 @@ setup( ], install_requires=[ "pyee~=9.0.4", - "websockets~=10.4", - "requests~=2.28.1" + "websockets~=11.0.3", + "requests~=2.28.1", + "urllib3~=1.26.14", ], python_requires=">=3.8" ) From 2bed2f6672180bb34f06f6648a14f2c131132b13 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 07:54:50 +0200 Subject: [PATCH 57/65] Fix bug in cancel_order_multi (both rest and websocket). --- bfxapi/rest/endpoints/rest_auth_endpoints.py | 8 ++++---- bfxapi/websocket/_client/bfx_websocket_inputs.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bfxapi/rest/endpoints/rest_auth_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py index 658e182..551389c 100644 --- a/bfxapi/rest/endpoints/rest_auth_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -121,12 +121,12 @@ class RestAuthEndpoints(Middleware): def cancel_order_multi(self, *, - ids: Optional[List[int]] = None, - cids: Optional[List[Tuple[int, str]]] = None, - gids: Optional[List[int]] = None, + id: Optional[List[int]] = None, + cid: Optional[List[Tuple[int, str]]] = None, + gid: Optional[List[int]] = None, all: bool = False) -> Notification[List[Order]]: body = { - "ids": ids, "cids": cids, "gids": gids, + "id": id, "cid": cid, "gid": gid, "all": all } diff --git a/bfxapi/websocket/_client/bfx_websocket_inputs.py b/bfxapi/websocket/_client/bfx_websocket_inputs.py index 1753f3a..b527b87 100644 --- a/bfxapi/websocket/_client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/_client/bfx_websocket_inputs.py @@ -64,12 +64,12 @@ class BfxWebSocketInputs: async def cancel_order_multi(self, *, - ids: Optional[List[int]] = None, - cids: Optional[List[Tuple[int, str]]] = None, - gids: Optional[List[int]] = None, + id: Optional[List[int]] = None, + cid: Optional[List[Tuple[int, str]]] = None, + gid: Optional[List[int]] = None, all: Optional[bool] = None) -> None: await self.__handle_websocket_input("oc_multi", { - "ids": ids, "cids": cids, "gids": gids, + "id": id, "cid": cid, "gid": gid, "all": all }) From 8c65ba54e93a9b97ef4f3c285fb714972094a81c Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 07:56:21 +0200 Subject: [PATCH 58/65] Rename property 'renew' to 'op_renew' in get_deposit_address. --- bfxapi/rest/endpoints/rest_auth_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bfxapi/rest/endpoints/rest_auth_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py index 551389c..d17ef66 100644 --- a/bfxapi/rest/endpoints/rest_auth_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -445,10 +445,10 @@ class RestAuthEndpoints(Middleware): def get_deposit_address(self, wallet: str, method: str, - renew: bool = False) -> Notification[DepositAddress]: + op_renew: bool = False) -> Notification[DepositAddress]: return _Notification[DepositAddress](serializers.DepositAddress) \ .parse(*self._post("auth/w/deposit/address", \ - body={ "wallet": wallet, "method": method, "renew": renew })) + body={ "wallet": wallet, "method": method, "op_renew": op_renew })) def generate_deposit_invoice(self, wallet: str, From 928772367807ca47d79c3441c52cca40a10297da Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 16:48:05 +0200 Subject: [PATCH 59/65] Fix several bugs in sub-package bfxapi.rest.endpoints. --- bfxapi/rest/endpoints/rest_auth_endpoints.py | 54 +++++++++---------- .../rest/endpoints/rest_merchant_endpoints.py | 2 +- .../rest/endpoints/rest_public_endpoints.py | 8 +-- .../websocket/_client/bfx_websocket_inputs.py | 2 +- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/bfxapi/rest/endpoints/rest_auth_endpoints.py b/bfxapi/rest/endpoints/rest_auth_endpoints.py index d17ef66..7029a08 100644 --- a/bfxapi/rest/endpoints/rest_auth_endpoints.py +++ b/bfxapi/rest/endpoints/rest_auth_endpoints.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Tuple, Union, Literal, Optional, Any +from typing import Dict, List, Tuple, Union, Literal, Optional from decimal import Decimal @@ -63,24 +63,22 @@ class RestAuthEndpoints(Middleware): def submit_order(self, type: str, symbol: str, - amount: Union[Decimal, float, str], + amount: Union[str, float, Decimal], + price: Union[str, float, Decimal], *, - price: Optional[Union[Decimal, float, str]] = None, lev: Optional[int] = None, - price_trailing: Optional[Union[Decimal, float, str]] = None, - price_aux_limit: Optional[Union[Decimal, float, str]] = None, - price_oco_stop: Optional[Union[Decimal, float, str]] = None, + price_trailing: Optional[Union[str, float, Decimal]] = None, + price_aux_limit: Optional[Union[str, float, Decimal]] = None, + price_oco_stop: Optional[Union[str, float, Decimal]] = None, gid: Optional[int] = None, cid: Optional[int] = None, - flags: Optional[int] = 0, - tif: Optional[str] = None, - meta: Optional[Dict[str, Any]] = None) -> Notification[Order]: + flags: Optional[int] = None, + tif: Optional[str] = None) -> Notification[Order]: body = { "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, "price_aux_limit": price_aux_limit, "price_oco_stop": price_oco_stop, "gid": gid, - "cid": cid, "flags": flags, "tif": tif, - "meta": meta + "cid": cid, "flags": flags, "tif": tif } return _Notification[Order](serializers.Order) \ @@ -89,16 +87,16 @@ class RestAuthEndpoints(Middleware): def update_order(self, id: int, *, - amount: Optional[Union[Decimal, float, str]] = None, - price: Optional[Union[Decimal, float, str]] = None, + amount: Optional[Union[str, float, Decimal]] = None, + price: Optional[Union[str, float, Decimal]] = None, cid: Optional[int] = None, cid_date: Optional[str] = None, gid: Optional[int] = None, - flags: Optional[int] = 0, + flags: Optional[int] = None, lev: Optional[int] = None, - delta: Optional[Union[Decimal, float, str]] = None, - price_aux_limit: Optional[Union[Decimal, float, str]] = None, - price_trailing: Optional[Union[Decimal, float, str]] = None, + delta: Optional[Union[str, float, Decimal]] = None, + price_aux_limit: Optional[Union[str, float, Decimal]] = None, + price_trailing: Optional[Union[str, float, Decimal]] = None, tif: Optional[str] = None) -> Notification[Order]: body = { "id": id, "amount": amount, "price": price, @@ -124,7 +122,7 @@ class RestAuthEndpoints(Middleware): id: Optional[List[int]] = None, cid: Optional[List[Tuple[int, str]]] = None, gid: Optional[List[int]] = None, - all: bool = False) -> Notification[List[Order]]: + all: Optional[bool] = None) -> Notification[List[Order]]: body = { "id": id, "cid": cid, "gid": gid, "all": all @@ -211,21 +209,21 @@ class RestAuthEndpoints(Middleware): def claim_position(self, id: int, *, - amount: Optional[Union[Decimal, float, str]] = None) -> Notification[PositionClaim]: + amount: Optional[Union[str, float, Decimal]] = None) -> Notification[PositionClaim]: return _Notification[PositionClaim](serializers.PositionClaim) \ .parse(*self._post("auth/w/position/claim", \ body={ "id": id, "amount": amount })) def increase_position(self, symbol: str, - amount: Union[Decimal, float, str]) -> Notification[PositionIncrease]: + amount: Union[str, float, Decimal]) -> Notification[PositionIncrease]: return _Notification[PositionIncrease](serializers.PositionIncrease) \ .parse(*self._post("auth/w/position/increase", \ body={ "symbol": symbol, "amount": amount })) def get_increase_position_info(self, symbol: str, - amount: Union[Decimal, float, str]) -> PositionIncreaseInfo: + amount: Union[str, float, Decimal]) -> PositionIncreaseInfo: return serializers.PositionIncreaseInfo \ .parse(*self._post("auth/r/position/increase/info", \ body={ "symbol": symbol, "amount": amount })) @@ -264,7 +262,7 @@ class RestAuthEndpoints(Middleware): def set_derivative_position_collateral(self, symbol: str, - collateral: Union[Decimal, float, str]) -> DerivativePositionCollateral: + collateral: Union[str, float, Decimal]) -> DerivativePositionCollateral: return serializers.DerivativePositionCollateral \ .parse(*(self._post("auth/w/deriv/collateral/set", \ body={ "symbol": symbol, "collateral": collateral })[0])) @@ -285,11 +283,11 @@ class RestAuthEndpoints(Middleware): def submit_funding_offer(self, type: str, symbol: str, - amount: Union[Decimal, float, str], - rate: Union[Decimal, float, str], + amount: Union[str, float, Decimal], + rate: Union[str, float, Decimal], period: int, *, - flags: Optional[int] = 0) -> Notification[FundingOffer]: + flags: Optional[int] = None) -> Notification[FundingOffer]: body = { "type": type, "symbol": symbol, "amount": amount, "rate": rate, "period": period, "flags": flags @@ -420,7 +418,7 @@ class RestAuthEndpoints(Middleware): to_wallet: str, currency: str, currency_to: str, - amount: Union[Decimal, float, str]) -> Notification[Transfer]: + amount: Union[str, float, Decimal]) -> Notification[Transfer]: body = { "from": from_wallet, "to": to_wallet, "currency": currency, "currency_to": currency_to, "amount": amount @@ -433,7 +431,7 @@ class RestAuthEndpoints(Middleware): wallet: str, method: str, address: str, - amount: Union[Decimal, float, str]) -> Notification[Withdrawal]: + amount: Union[str, float, Decimal]) -> Notification[Withdrawal]: body = { "wallet": wallet, "method": method, "address": address, "amount": amount @@ -453,7 +451,7 @@ class RestAuthEndpoints(Middleware): def generate_deposit_invoice(self, wallet: str, currency: str, - amount: Union[Decimal, float, str]) -> LightningNetworkInvoice: + amount: Union[str, float, Decimal]) -> LightningNetworkInvoice: return serializers.LightningNetworkInvoice \ .parse(*self._post("auth/w/deposit/invoice", \ body={ "wallet": wallet, "currency": currency, "amount": amount })) diff --git a/bfxapi/rest/endpoints/rest_merchant_endpoints.py b/bfxapi/rest/endpoints/rest_merchant_endpoints.py index d34028b..848a387 100644 --- a/bfxapi/rest/endpoints/rest_merchant_endpoints.py +++ b/bfxapi/rest/endpoints/rest_merchant_endpoints.py @@ -30,7 +30,7 @@ _CustomerInfo = TypedDict("_CustomerInfo", { class RestMerchantEndpoints(Middleware): #pylint: disable-next=too-many-arguments def submit_invoice(self, - amount: Union[Decimal, float, str], + amount: Union[str, float, Decimal], currency: str, order_id: str, customer_info: _CustomerInfo, diff --git a/bfxapi/rest/endpoints/rest_public_endpoints.py b/bfxapi/rest/endpoints/rest_public_endpoints.py index 5549730..db899ed 100644 --- a/bfxapi/rest/endpoints/rest_public_endpoints.py +++ b/bfxapi/rest/endpoints/rest_public_endpoints.py @@ -270,9 +270,9 @@ class RestPublicEndpoints(Middleware): def get_trading_market_average_price(self, symbol: str, - amount: Union[Decimal, float, str], + amount: Union[str, float, Decimal], *, - price_limit: Optional[Union[Decimal, float, str]] = None + price_limit: Optional[Union[str, float, Decimal]] = None ) -> TradingMarketAveragePrice: return serializers.TradingMarketAveragePrice.parse(*self._post("calc/trade/avg", body={ "symbol": symbol, "amount": amount, "price_limit": price_limit @@ -280,10 +280,10 @@ class RestPublicEndpoints(Middleware): def get_funding_market_average_price(self, symbol: str, - amount: Union[Decimal, float, str], + amount: Union[str, float, Decimal], period: int, *, - rate_limit: Optional[Union[Decimal, float, str]] = None + rate_limit: Optional[Union[str, float, Decimal]] = None ) -> FundingMarketAveragePrice: return serializers.FundingMarketAveragePrice.parse(*self._post("calc/trade/avg", body={ "symbol": symbol, "amount": amount, "period": period, "rate_limit": rate_limit diff --git a/bfxapi/websocket/_client/bfx_websocket_inputs.py b/bfxapi/websocket/_client/bfx_websocket_inputs.py index b527b87..5373d7a 100644 --- a/bfxapi/websocket/_client/bfx_websocket_inputs.py +++ b/bfxapi/websocket/_client/bfx_websocket_inputs.py @@ -29,7 +29,7 @@ class BfxWebSocketInputs: "type": type, "symbol": symbol, "amount": amount, "price": price, "lev": lev, "price_trailing": price_trailing, "price_aux_limit": price_aux_limit, "price_oco_stop": price_oco_stop, "gid": gid, - "cid": cid, "flags": flags, "tif": tif, + "cid": cid, "flags": flags, "tif": tif }) async def update_order(self, From 1ec6c494286f8266772e0fa42a5f214f3a8f9898 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 17:46:38 +0200 Subject: [PATCH 60/65] Rewrite all rest examples according to v3.0.0b3's changes. --- bfxapi/rest/middleware/middleware.py | 4 +- examples/rest/auth/claim_position.py | 8 ++-- examples/rest/auth/get_wallets.py | 38 ++++++++++++------- .../set_derivative_position_collateral.py | 33 ++++++++-------- examples/rest/auth/submit_funding_offer.py | 13 ++----- examples/rest/auth/submit_order.py | 18 ++------- examples/rest/auth/toggle_keep_funding.py | 9 +---- examples/rest/merchant/settings.py | 27 ++++++------- examples/rest/merchant/submit_invoice.py | 22 +++++------ examples/rest/public/book.py | 21 ++++++---- examples/rest/public/conf.py | 20 ++++------ examples/rest/public/get_candles_hist.py | 11 ++++-- examples/rest/public/pulse_endpoints.py | 15 +++++--- .../rest/public/rest_calculation_endpoints.py | 24 ++++++------ examples/rest/public/trades.py | 17 +++++---- 15 files changed, 141 insertions(+), 139 deletions(-) diff --git a/bfxapi/rest/middleware/middleware.py b/bfxapi/rest/middleware/middleware.py index f279f30..b4c9916 100644 --- a/bfxapi/rest/middleware/middleware.py +++ b/bfxapi/rest/middleware/middleware.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Optional, Any -from enum import Enum +from enum import IntEnum from http import HTTPStatus @@ -15,7 +15,7 @@ from ..._utils.json_decoder import JSONDecoder if TYPE_CHECKING: from requests.sessions import _Params -class _Error(Enum): +class _Error(IntEnum): ERR_UNK = 10000 ERR_GENERIC = 10001 ERR_PARAMS = 10020 diff --git a/examples/rest/auth/claim_position.py b/examples/rest/auth/claim_position.py index 8243600..d0da7f9 100644 --- a/examples/rest/auth/claim_position.py +++ b/examples/rest/auth/claim_position.py @@ -2,18 +2,18 @@ import os -from bfxapi import Client, REST_HOST - +from bfxapi import Client from bfxapi.types import Notification, PositionClaim bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) # Claims all active positions for position in bfx.rest.auth.get_positions(): - notification: Notification[PositionClaim] = bfx.rest.auth.claim_position(position.position_id) + notification: Notification[PositionClaim] = bfx.rest.auth.claim_position( + position.position_id + ) claim: PositionClaim = notification.data print(f"Position: {position} | PositionClaim: {claim}") diff --git a/examples/rest/auth/get_wallets.py b/examples/rest/auth/get_wallets.py index b0c5aae..67696fc 100644 --- a/examples/rest/auth/get_wallets.py +++ b/examples/rest/auth/get_wallets.py @@ -1,16 +1,19 @@ # python -c "import examples.rest.auth.get_wallets" import os - from typing import List -from bfxapi import Client, REST_HOST - -from bfxapi.types import Wallet, Transfer, DepositAddress, \ - LightningNetworkInvoice, Withdrawal, Notification +from bfxapi import Client +from bfxapi.types import ( + DepositAddress, + LightningNetworkInvoice, + Notification, + Transfer, + Wallet, + Withdrawal, +) bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) @@ -20,26 +23,35 @@ wallets: List[Wallet] = bfx.rest.auth.get_wallets() # Transfers funds (0.001 ETH) from exchange wallet to funding wallet A: Notification[Transfer] = bfx.rest.auth.transfer_between_wallets( - from_wallet="exchange", to_wallet="funding", currency="ETH", - currency_to="ETH", amount=0.001) + from_wallet="exchange", + to_wallet="funding", + currency="ETH", + currency_to="ETH", + amount=0.001, +) print("Transfer:", A.data) # Retrieves the deposit address for bitcoin currency in exchange wallet. B: Notification[DepositAddress] = bfx.rest.auth.get_deposit_address( - wallet="exchange", method="bitcoin", renew=False) + wallet="exchange", method="bitcoin", op_renew=False +) print("Deposit address:", B.data) # Generates a lightning network deposit invoice C: Notification[LightningNetworkInvoice] = bfx.rest.auth.generate_deposit_invoice( - wallet="funding", currency="LNX", amount=0.001) + wallet="funding", currency="LNX", amount=0.001 +) print("Lightning network invoice:", C.data) -# Withdraws 1.0 UST from user's exchange wallet to address 0x742d35Cc6634C0532925a3b844Bc454e4438f44e +# Withdraws 1.0 UST from user's exchange wallet to address 0x742d35... D: Notification[Withdrawal] = bfx.rest.auth.submit_wallet_withdrawal( - wallet="exchange", method="tetheruse", address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e", - amount=1.0) + wallet="exchange", + method="tetheruse", + address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e", + amount=1.0, +) print("Withdrawal:", D.data) diff --git a/examples/rest/auth/set_derivative_position_collateral.py b/examples/rest/auth/set_derivative_position_collateral.py index 7365b83..37a8002 100644 --- a/examples/rest/auth/set_derivative_position_collateral.py +++ b/examples/rest/auth/set_derivative_position_collateral.py @@ -1,36 +1,39 @@ -# python -c "import examples.rest.auth.set_derivatives_position_collateral" +# python -c "import examples.rest.auth.set_derivative_position_collateral" import os -from bfxapi import Client, REST_HOST - -from bfxapi.types import DerivativePositionCollateral, DerivativePositionCollateralLimits +from bfxapi import Client +from bfxapi.types import ( + DerivativePositionCollateral, + DerivativePositionCollateralLimits, +) bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) submit_order_notification = bfx.rest.auth.submit_order( - type="LIMIT", - symbol="tBTCF0:USTF0", - amount="0.015", - price="16700", - lev=10 + type="LIMIT", symbol="tBTCF0:USTF0", amount="0.015", price="16700", lev=10 ) print("New Order:", submit_order_notification.data) # Update the amount of collateral for tBTCF0:USTF0 derivative position -derivative_position_collateral: DerivativePositionCollateral = \ - bfx.rest.auth.set_derivative_position_collateral(symbol="tBTCF0:USTF0", collateral=50.0) +derivative_position_collateral: DerivativePositionCollateral = ( + bfx.rest.auth.set_derivative_position_collateral( + symbol="tBTCF0:USTF0", collateral=50.0 + ) +) print("Status:", bool(derivative_position_collateral.status)) # Calculate the minimum and maximum collateral that can be assigned to tBTCF0:USTF0. -derivative_position_collateral_limits: DerivativePositionCollateralLimits = \ +derivative_position_collateral_limits: DerivativePositionCollateralLimits = ( bfx.rest.auth.get_derivative_position_collateral_limits(symbol="tBTCF0:USTF0") +) -print(f"Minimum collateral: {derivative_position_collateral_limits.min_collateral} | " \ - f"Maximum collateral: {derivative_position_collateral_limits.max_collateral}") +print( + f"Minimum collateral: {derivative_position_collateral_limits.min_collateral} | " + f"Maximum collateral: {derivative_position_collateral_limits.max_collateral}" +) diff --git a/examples/rest/auth/submit_funding_offer.py b/examples/rest/auth/submit_funding_offer.py index 2a29b27..d390915 100644 --- a/examples/rest/auth/submit_funding_offer.py +++ b/examples/rest/auth/submit_funding_offer.py @@ -2,24 +2,17 @@ import os -from bfxapi import Client, REST_HOST -from bfxapi.types import Notification, FundingOffer -from bfxapi.enums import FundingOfferType, Flag +from bfxapi import Client +from bfxapi.types import FundingOffer, Notification bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) # Submit a new funding offer notification: Notification[FundingOffer] = bfx.rest.auth.submit_funding_offer( - type=FundingOfferType.LIMIT, - symbol="fUSD", - amount=123.45, - rate=0.001, - period=2, - flags=Flag.HIDDEN + type="LIMIT", symbol="fUSD", amount=123.45, rate=0.001, period=2 ) print("Funding Offer notification:", notification) diff --git a/examples/rest/auth/submit_order.py b/examples/rest/auth/submit_order.py index 8fc2830..bf4f899 100644 --- a/examples/rest/auth/submit_order.py +++ b/examples/rest/auth/submit_order.py @@ -2,23 +2,17 @@ import os -from bfxapi import Client, REST_HOST +from bfxapi import Client from bfxapi.types import Notification, Order -from bfxapi.enums import OrderType, Flag bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) # Submit a new order submit_order_notification: Notification[Order] = bfx.rest.auth.submit_order( - type=OrderType.EXCHANGE_LIMIT, - symbol="tBTCUST", - amount=0.015, - price=10000, - flags=Flag.HIDDEN + Flag.OCO + Flag.CLOSE + type="EXCHANGE LIMIT", symbol="tBTCUST", amount=0.015, price=10000 ) print("Submit order notification:", submit_order_notification) @@ -27,16 +21,12 @@ order: Order = submit_order_notification.data # Update its amount and its price update_order_notification: Notification[Order] = bfx.rest.auth.update_order( - id=order.id, - amount=0.020, - price=10150 + id=order.id, amount=0.020, price=10150 ) print("Update order notification:", update_order_notification) # Cancel it by its ID -cancel_order_notification: Notification[Order] = bfx.rest.auth.cancel_order( - id=order.id -) +cancel_order_notification: Notification[Order] = bfx.rest.auth.cancel_order(id=order.id) print("Cancel order notification:", cancel_order_notification) diff --git a/examples/rest/auth/toggle_keep_funding.py b/examples/rest/auth/toggle_keep_funding.py index 8a1880d..dbacdcf 100644 --- a/examples/rest/auth/toggle_keep_funding.py +++ b/examples/rest/auth/toggle_keep_funding.py @@ -1,15 +1,12 @@ # python -c "import examples.rest.auth.toggle_keep_funding" import os - from typing import List -from bfxapi import Client, REST_HOST - +from bfxapi import Client from bfxapi.types import FundingLoan, Notification bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) @@ -18,9 +15,7 @@ loans: List[FundingLoan] = bfx.rest.auth.get_funding_loans(symbol="fUSD") # Set every loan's keep funding status to (1: , 2: ) notification: Notification[None] = bfx.rest.auth.toggle_keep_funding( - type="loan", - ids=[ loan.id for loan in loans ], - changes={ loan.id: 2 for loan in loans } + type="loan", ids=[loan.id for loan in loans], changes={loan.id: 2 for loan in loans} ) print("Toggle keep funding notification:", notification) diff --git a/examples/rest/merchant/settings.py b/examples/rest/merchant/settings.py index 4f974b7..e82008e 100644 --- a/examples/rest/merchant/settings.py +++ b/examples/rest/merchant/settings.py @@ -2,27 +2,28 @@ import os -from bfxapi import Client, REST_HOST - -from bfxapi.rest.enums import MerchantSettingsKey +from bfxapi import Client bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) -if not bfx.rest.merchant.set_merchant_settings(MerchantSettingsKey.RECOMMEND_STORE, 1): - print(f"Cannot set <{MerchantSettingsKey.RECOMMEND_STORE}> to <1>.") +if not bfx.rest.merchant.set_merchant_settings("bfx_pay_recommend_store", 1): + print("Cannot set to <1>.") -print(f"The current <{MerchantSettingsKey.PREFERRED_FIAT}> value is:", - bfx.rest.merchant.get_merchant_settings(MerchantSettingsKey.PREFERRED_FIAT)) +print( + "The current value is:", + bfx.rest.merchant.get_merchant_settings("bfx_pay_preferred_fiat"), +) -settings = bfx.rest.merchant.list_merchant_settings([ - MerchantSettingsKey.DUST_BALANCE_UI, - MerchantSettingsKey.MERCHANT_CUSTOMER_SUPPORT_URL, - MerchantSettingsKey.MERCHANT_UNDERPAID_THRESHOLD -]) +settings = bfx.rest.merchant.list_merchant_settings( + [ + "bfx_pay_dust_balance_ui", + "bfx_pay_merchant_customer_support_url", + "bfx_pay_merchant_underpaid_threshold", + ] +) for key, value in settings.items(): print(f"<{key}>:", value) diff --git a/examples/rest/merchant/submit_invoice.py b/examples/rest/merchant/submit_invoice.py index 446a1c3..c2962b0 100644 --- a/examples/rest/merchant/submit_invoice.py +++ b/examples/rest/merchant/submit_invoice.py @@ -2,12 +2,10 @@ import os -from bfxapi import Client, REST_HOST - +from bfxapi import Client from bfxapi.types import InvoiceSubmission bfx = Client( - rest_host=REST_HOST, api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET") ) @@ -20,7 +18,7 @@ customer_info = { "residStreet": "5-6 Leicester Square", "residBuildingNo": "23 A", "fullName": "John Doe", - "email": "john@example.com" + "email": "john@example.com", } invoice: InvoiceSubmission = bfx.rest.merchant.submit_invoice( @@ -29,17 +27,19 @@ invoice: InvoiceSubmission = bfx.rest.merchant.submit_invoice( order_id="test", customer_info=customer_info, pay_currencies=["ETH"], - duration=86400 * 10 + duration=86400, ) print("Invoice submission:", invoice) -print(bfx.rest.merchant.complete_invoice( - id=invoice.id, - pay_currency="ETH", - deposit_id=1 -)) +print( + bfx.rest.merchant.complete_invoice(id=invoice.id, pay_currency="ETH", deposit_id=1) +) print(bfx.rest.merchant.get_invoices(limit=25)) -print(bfx.rest.merchant.get_invoices_paginated(page=1, page_size=60, sort="asc", sort_field="t")) +print( + bfx.rest.merchant.get_invoices_paginated( + page=1, page_size=60, sort="asc", sort_field="t" + ) +) diff --git a/examples/rest/public/book.py b/examples/rest/public/book.py index 8cb11f8..e352b99 100644 --- a/examples/rest/public/book.py +++ b/examples/rest/public/book.py @@ -2,14 +2,19 @@ from typing import List -from bfxapi import Client, PUB_REST_HOST +from bfxapi import Client +from bfxapi.types import ( + FundingCurrencyBook, + FundingCurrencyRawBook, + TradingPairBook, + TradingPairRawBook, +) -from bfxapi.types import TradingPairBook, TradingPairRawBook, \ - FundingCurrencyBook, FundingCurrencyRawBook +bfx = Client() -bfx = Client(rest_host=PUB_REST_HOST) - -t_book: List[TradingPairBook] = bfx.rest.public.get_t_book("tBTCUSD", precision="P0", len=25) +t_book: List[TradingPairBook] = bfx.rest.public.get_t_book( + "tBTCUSD", precision="P0", len=25 +) print("25 price points of tBTCUSD order book (with precision P0):", t_book) @@ -17,7 +22,9 @@ t_raw_book: List[TradingPairRawBook] = bfx.rest.public.get_t_raw_book("tBTCUSD") print("tBTCUSD raw order book:", t_raw_book) -f_book: List[FundingCurrencyBook] = bfx.rest.public.get_f_book("fUSD", precision="P0", len=25) +f_book: List[FundingCurrencyBook] = bfx.rest.public.get_f_book( + "fUSD", precision="P0", len=25 +) print("25 price points of fUSD order book (with precision P0):", f_book) diff --git a/examples/rest/public/conf.py b/examples/rest/public/conf.py index 431eb26..0efbfbd 100644 --- a/examples/rest/public/conf.py +++ b/examples/rest/public/conf.py @@ -1,18 +1,14 @@ # python -c "import examples.rest.public.conf" -from bfxapi import Client, PUB_REST_HOST +from bfxapi import Client -from bfxapi.rest.enums import Config +bfx = Client() -bfx = Client(rest_host=PUB_REST_HOST) +# Prints a map from symbols to their API symbols +print(bfx.rest.public.conf("pub:map:currency:sym")) -print("Available configs:", [ config.value for config in Config ]) +# Prints all the available exchange trading pairs +print(bfx.rest.public.conf("pub:list:pair:exchange")) -# Prints a map from symbols to their API symbols (pub:map:currency:sym) -print (bfx.rest.public.conf(Config.MAP_CURRENCY_SYM)) - -# Prints all the available exchange trading pairs (pub:list:pair:exchange) -print(bfx.rest.public.conf(Config.LIST_PAIR_EXCHANGE)) - -# Prints all the available funding currencies (pub:list:currency) -print(bfx.rest.public.conf(Config.LIST_CURRENCY)) +# Prints all the available funding currencies +print(bfx.rest.public.conf("pub:list:currency")) diff --git a/examples/rest/public/get_candles_hist.py b/examples/rest/public/get_candles_hist.py index 12588b1..ed7a80b 100644 --- a/examples/rest/public/get_candles_hist.py +++ b/examples/rest/public/get_candles_hist.py @@ -1,11 +1,14 @@ # python -c "import examples.rest.public.get_candles_hist" -from bfxapi import Client, PUB_REST_HOST +from bfxapi import Client -bfx = Client(rest_host=PUB_REST_HOST) +bfx = Client() print(f"Candles: {bfx.rest.public.get_candles_hist(symbol='tBTCUSD')}") # Be sure to specify a period or aggregated period when retrieving funding candles. -# If you wish to mimic the candles found in the UI, use the following setup to aggregate all funding candles: a30:p2:p30 -print(f"Candles: {bfx.rest.public.get_candles_hist(tf='15m', symbol='fUSD:a30:p2:p30')}") +# If you wish to mimic the candles found in the UI, use the following setup +# to aggregate all funding candles: a30:p2:p30 +print( + f"Candles: {bfx.rest.public.get_candles_hist(tf='15m', symbol='fUSD:a30:p2:p30')}" +) diff --git a/examples/rest/public/pulse_endpoints.py b/examples/rest/public/pulse_endpoints.py index 3784500..7fb2bca 100644 --- a/examples/rest/public/pulse_endpoints.py +++ b/examples/rest/public/pulse_endpoints.py @@ -1,20 +1,20 @@ # python -c "import examples.rest.public.pulse_endpoints" import datetime - from typing import List -from bfxapi import Client, PUB_REST_HOST - +from bfxapi import Client from bfxapi.types import PulseMessage, PulseProfile -bfx = Client(rest_host=PUB_REST_HOST) +bfx = Client() # POSIX timestamp in milliseconds (check https://currentmillis.com/) end = datetime.datetime(2020, 5, 2).timestamp() * 1000 # Retrieves 25 pulse messages up to 2020/05/02 -messages: List[PulseMessage] = bfx.rest.public.get_pulse_message_history(end=end, limit=25) +messages: List[PulseMessage] = bfx.rest.public.get_pulse_message_history( + end=end, limit=25 +) for message in messages: print(f"Message author: {message.profile.nickname} ({message.profile.puid})") @@ -23,4 +23,7 @@ for message in messages: profile: PulseProfile = bfx.rest.public.get_pulse_profile_details("News") URL = profile.picture.replace("size", "small") -print(f"<{profile.nickname}>'s profile picture: https://s3-eu-west-1.amazonaws.com/bfx-pub/{URL}") +print( + f"<{profile.nickname}>'s profile picture:" + f" https://s3-eu-west-1.amazonaws.com/bfx-pub/{URL}" +) diff --git a/examples/rest/public/rest_calculation_endpoints.py b/examples/rest/public/rest_calculation_endpoints.py index 88fba15..0a07945 100644 --- a/examples/rest/public/rest_calculation_endpoints.py +++ b/examples/rest/public/rest_calculation_endpoints.py @@ -1,24 +1,22 @@ # python -c "import examples.rest.public.rest_calculation_endpoints" -from bfxapi import Client, PUB_REST_HOST +from bfxapi import Client +from bfxapi.types import FundingMarketAveragePrice, FxRate, TradingMarketAveragePrice -from bfxapi.types import TradingMarketAveragePrice, FundingMarketAveragePrice, FxRate +bfx = Client() -bfx = Client(rest_host=PUB_REST_HOST) - -trading_market_average_price: TradingMarketAveragePrice = bfx.rest.public.get_trading_market_average_price( - symbol="tBTCUSD", - amount=-100, - price_limit=20000.5 +trading_market_average_price: TradingMarketAveragePrice = ( + bfx.rest.public.get_trading_market_average_price( + symbol="tBTCUSD", amount=-100, price_limit=20000.5 + ) ) print("Average execution price for tBTCUSD:", trading_market_average_price.price_avg) -funding_market_average_price: FundingMarketAveragePrice = bfx.rest.public.get_funding_market_average_price( - symbol="fUSD", - amount=100, - period=2, - rate_limit=0.00015 +funding_market_average_price: FundingMarketAveragePrice = ( + bfx.rest.public.get_funding_market_average_price( + symbol="fUSD", amount=100, period=2, rate_limit=0.00015 + ) ) print("Average execution rate for fUSD:", funding_market_average_price.rate_avg) diff --git a/examples/rest/public/trades.py b/examples/rest/public/trades.py index d83ff2b..c37b9da 100644 --- a/examples/rest/public/trades.py +++ b/examples/rest/public/trades.py @@ -2,18 +2,19 @@ from typing import List -from bfxapi import Client, PUB_REST_HOST -from bfxapi.types import TradingPairTrade, FundingCurrencyTrade -from bfxapi.rest.enums import Sort +from bfxapi import Client +from bfxapi.types import FundingCurrencyTrade, TradingPairTrade -bfx = Client(rest_host=PUB_REST_HOST) +bfx = Client() -t_trades: List[TradingPairTrade] = bfx.rest.public.get_t_trades("tBTCUSD", \ - limit=15, sort=Sort.ASCENDING) +t_trades: List[TradingPairTrade] = bfx.rest.public.get_t_trades( + "tBTCUSD", limit=15, sort=+1 +) print("Latest 15 trades for tBTCUSD (in ascending order):", t_trades) -f_trades: List[FundingCurrencyTrade] = bfx.rest.public.get_f_trades("fUSD", \ - limit=15, sort=Sort.DESCENDING) +f_trades: List[FundingCurrencyTrade] = bfx.rest.public.get_f_trades( + "fUSD", limit=15, sort=-1 +) print("Latest 15 trades for fUSD (in descending order):", f_trades) From 1accf92c57baa6a83cda8d2da11841740bfc7c30 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 17:59:44 +0200 Subject: [PATCH 61/65] Rewrite all websocket examples according to v3.0.0b3's changes. --- examples/websocket/auth/submit_order.py | 16 ++--- examples/websocket/auth/wallets.py | 7 +- .../websocket/public/derivatives_status.py | 9 +-- examples/websocket/public/order_book.py | 69 +++++++++++-------- examples/websocket/public/raw_order_book.py | 69 +++++++++++-------- examples/websocket/public/ticker.py | 11 +-- examples/websocket/public/trades.py | 14 ++-- 7 files changed, 111 insertions(+), 84 deletions(-) diff --git a/examples/websocket/auth/submit_order.py b/examples/websocket/auth/submit_order.py index a65935a..771095e 100644 --- a/examples/websocket/auth/submit_order.py +++ b/examples/websocket/auth/submit_order.py @@ -2,39 +2,39 @@ import os -from bfxapi import Client, WSS_HOST -from bfxapi.enums import Error, OrderType +from bfxapi import Client from bfxapi.types import Notification, Order bfx = Client( - wss_host=WSS_HOST, api_key=os.getenv("BFX_API_KEY"), - api_secret=os.getenv("BFX_API_SECRET") + api_secret=os.getenv("BFX_API_SECRET"), ) + @bfx.wss.on("authenticated") async def on_authenticated(event): print(f"Authentication: {event}") await bfx.wss.inputs.submit_order( - type=OrderType.EXCHANGE_LIMIT, - symbol="tBTCUSD", - amount="0.1", - price="10000.0" + type="EXCHANGE LIMIT", symbol="tBTCUSD", amount=0.165212, price=30264.0 ) print("The order has been sent.") + @bfx.wss.on("on-req-notification") async def on_notification(notification: Notification[Order]): print(f"Notification: {notification}.") + @bfx.wss.on("order_new") async def on_order_new(order_new: Order): print(f"Order new: {order_new}") + @bfx.wss.on("subscribed") def on_subscribed(subscription): print(f"Subscription successful for <{subscription}>.") + bfx.wss.run() diff --git a/examples/websocket/auth/wallets.py b/examples/websocket/auth/wallets.py index a8d40ed..29eaaf8 100644 --- a/examples/websocket/auth/wallets.py +++ b/examples/websocket/auth/wallets.py @@ -1,19 +1,18 @@ # python -c "import examples.websocket.auth.wallets" import os - from typing import List from bfxapi import Client -from bfxapi.enums import Error from bfxapi.types import Wallet bfx = Client( api_key=os.getenv("BFX_API_KEY"), api_secret=os.getenv("BFX_API_SECRET"), - filters=["wallet"] + filters=["wallet"], ) + @bfx.wss.on("wallet_snapshot") def on_wallet_snapshot(wallets: List[Wallet]): for wallet in wallets: @@ -21,8 +20,10 @@ def on_wallet_snapshot(wallets: List[Wallet]): print(f"Available balance: {wallet.available_balance}") print(f"Wallet trade details: {wallet.trade_details}") + @bfx.wss.on("wallet_update") def on_wallet_update(wallet: Wallet): print(f"Wallet update: {wallet}") + bfx.wss.run() diff --git a/examples/websocket/public/derivatives_status.py b/examples/websocket/public/derivatives_status.py index 982d3cb..31e8368 100644 --- a/examples/websocket/public/derivatives_status.py +++ b/examples/websocket/public/derivatives_status.py @@ -1,19 +1,20 @@ # python -c "import examples.websocket.public.derivatives_status" -from bfxapi import Client, PUB_WSS_HOST +from bfxapi import Client from bfxapi.types import DerivativesStatus from bfxapi.websocket.subscriptions import Status -from bfxapi.websocket.enums import Error, Channel +bfx = Client() -bfx = Client(wss_host=PUB_WSS_HOST) @bfx.wss.on("derivatives_status_update") def on_derivatives_status_update(subscription: Status, data: DerivativesStatus): print(f"{subscription}:", data) + @bfx.wss.on("open") async def on_open(): - await bfx.wss.subscribe(Channel.STATUS, key="deriv:tBTCF0:USTF0") + await bfx.wss.subscribe("status", key="deriv:tBTCF0:USTF0") + bfx.wss.run() diff --git a/examples/websocket/public/order_book.py b/examples/websocket/public/order_book.py index 5201ed8..c79ade7 100644 --- a/examples/websocket/public/order_book.py +++ b/examples/websocket/public/order_book.py @@ -1,27 +1,21 @@ # python -c "import examples.websocket.public.order_book" -from collections import OrderedDict - -from typing import List, Dict - import zlib +from collections import OrderedDict +from typing import Dict, List -from bfxapi import Client, PUB_WSS_HOST - +from bfxapi import Client from bfxapi.types import TradingPairBook from bfxapi.websocket.subscriptions import Book -from bfxapi.websocket.enums import Channel, Error + class OrderBook: def __init__(self, symbols: List[str]): self.__order_book = { - symbol: { - "bids": OrderedDict(), "asks": OrderedDict() - } for symbol in symbols + symbol: {"bids": OrderedDict(), "asks": OrderedDict()} for symbol in symbols } - self.cooldown: Dict[str, bool] = \ - { symbol: False for symbol in symbols } + self.cooldown: Dict[str, bool] = {symbol: False for symbol in symbols} def update(self, symbol: str, data: TradingPairBook) -> None: price, count, amount = data.price, data.count, data.amount @@ -30,9 +24,9 @@ class OrderBook: if count > 0: self.__order_book[symbol][kind][price] = { - "price": price, + "price": price, "count": count, - "amount": amount + "amount": amount, } if count == 0: @@ -40,23 +34,31 @@ class OrderBook: del self.__order_book[symbol][kind][price] def verify(self, symbol: str, checksum: int) -> bool: - values: List[int] = [ ] + values: List[int] = [] - bids = sorted([ (data["price"], data["count"], data["amount"]) \ - for _, data in self.__order_book[symbol]["bids"].items() ], - key=lambda data: -data[0]) + bids = sorted( + [ + (data["price"], data["count"], data["amount"]) + for _, data in self.__order_book[symbol]["bids"].items() + ], + key=lambda data: -data[0], + ) - asks = sorted([ (data["price"], data["count"], data["amount"]) \ - for _, data in self.__order_book[symbol]["asks"].items() ], - key=lambda data: data[0]) + asks = sorted( + [ + (data["price"], data["count"], data["amount"]) + for _, data in self.__order_book[symbol]["asks"].items() + ], + key=lambda data: data[0], + ) if len(bids) < 25 or len(asks) < 25: raise AssertionError("Not enough bids or asks (need at least 25).") for _i in range(25): bid, ask = bids[_i], asks[_i] - values.extend([ bid[0], bid[2] ]) - values.extend([ ask[0], ask[2] ]) + values.extend([bid[0], bid[2]]) + values.extend([ask[0], ask[2]]) local = ":".join(str(value) for value in values) @@ -64,30 +66,36 @@ class OrderBook: return crc32 == checksum -SYMBOLS = [ "tLTCBTC", "tETHUSD", "tETHBTC" ] + +SYMBOLS = ["tLTCBTC", "tETHUSD", "tETHBTC"] order_book = OrderBook(symbols=SYMBOLS) -bfx = Client(wss_host=PUB_WSS_HOST) +bfx = Client() + @bfx.wss.on("open") async def on_open(): for symbol in SYMBOLS: - await bfx.wss.subscribe(Channel.BOOK, symbol=symbol) + await bfx.wss.subscribe("book", symbol=symbol) + @bfx.wss.on("subscribed") def on_subscribed(subscription): - print(f"Subscription successful for pair <{subscription['pair']}>") + print(f"Subscription successful for symbol <{subscription['symbol']}>") + @bfx.wss.on("t_book_snapshot") def on_t_book_snapshot(subscription: Book, snapshot: List[TradingPairBook]): for data in snapshot: order_book.update(subscription["symbol"], data) + @bfx.wss.on("t_book_update") def on_t_book_update(subscription: Book, data: TradingPairBook): order_book.update(subscription["symbol"], data) + @bfx.wss.on("checksum") async def on_checksum(subscription: Book, value: int): symbol = subscription["symbol"] @@ -95,11 +103,14 @@ async def on_checksum(subscription: Book, value: int): if order_book.verify(symbol, value): order_book.cooldown[symbol] = False elif not order_book.cooldown[symbol]: - print("Mismatch between local and remote checksums: " - f"restarting book for symbol <{symbol}>...") + print( + "Mismatch between local and remote checksums: " + f"restarting book for symbol <{symbol}>..." + ) await bfx.wss.resubscribe(sub_id=subscription["sub_id"]) order_book.cooldown[symbol] = True + bfx.wss.run() diff --git a/examples/websocket/public/raw_order_book.py b/examples/websocket/public/raw_order_book.py index dedd291..fd6ebcc 100644 --- a/examples/websocket/public/raw_order_book.py +++ b/examples/websocket/public/raw_order_book.py @@ -1,27 +1,21 @@ # python -c "import examples.websocket.public.raw_order_book" -from collections import OrderedDict - -from typing import List, Dict - import zlib +from collections import OrderedDict +from typing import Dict, List -from bfxapi import Client, PUB_WSS_HOST - +from bfxapi import Client from bfxapi.types import TradingPairRawBook from bfxapi.websocket.subscriptions import Book -from bfxapi.websocket.enums import Channel, Error + class RawOrderBook: def __init__(self, symbols: List[str]): self.__raw_order_book = { - symbol: { - "bids": OrderedDict(), "asks": OrderedDict() - } for symbol in symbols + symbol: {"bids": OrderedDict(), "asks": OrderedDict()} for symbol in symbols } - self.cooldown: Dict[str, bool] = \ - { symbol: False for symbol in symbols } + self.cooldown: Dict[str, bool] = {symbol: False for symbol in symbols} def update(self, symbol: str, data: TradingPairRawBook) -> None: order_id, price, amount = data.order_id, data.price, data.amount @@ -31,8 +25,8 @@ class RawOrderBook: if price > 0: self.__raw_order_book[symbol][kind][order_id] = { "order_id": order_id, - "price": price, - "amount": amount + "price": price, + "amount": amount, } if price == 0: @@ -40,23 +34,31 @@ class RawOrderBook: del self.__raw_order_book[symbol][kind][order_id] def verify(self, symbol: str, checksum: int) -> bool: - values: List[int] = [ ] + values: List[int] = [] - bids = sorted([ (data["order_id"], data["price"], data["amount"]) \ - for _, data in self.__raw_order_book[symbol]["bids"].items() ], - key=lambda data: (-data[1], data[0])) + bids = sorted( + [ + (data["order_id"], data["price"], data["amount"]) + for _, data in self.__raw_order_book[symbol]["bids"].items() + ], + key=lambda data: (-data[1], data[0]), + ) - asks = sorted([ (data["order_id"], data["price"], data["amount"]) \ - for _, data in self.__raw_order_book[symbol]["asks"].items() ], - key=lambda data: (data[1], data[0])) + asks = sorted( + [ + (data["order_id"], data["price"], data["amount"]) + for _, data in self.__raw_order_book[symbol]["asks"].items() + ], + key=lambda data: (data[1], data[0]), + ) if len(bids) < 25 or len(asks) < 25: raise AssertionError("Not enough bids or asks (need at least 25).") for _i in range(25): bid, ask = bids[_i], asks[_i] - values.extend([ bid[0], bid[2] ]) - values.extend([ ask[0], ask[2] ]) + values.extend([bid[0], bid[2]]) + values.extend([ask[0], ask[2]]) local = ":".join(str(value) for value in values) @@ -64,30 +66,36 @@ class RawOrderBook: return crc32 == checksum -SYMBOLS = [ "tLTCBTC", "tETHUSD", "tETHBTC" ] + +SYMBOLS = ["tLTCBTC", "tETHUSD", "tETHBTC"] raw_order_book = RawOrderBook(symbols=SYMBOLS) -bfx = Client(wss_host=PUB_WSS_HOST) +bfx = Client() + @bfx.wss.on("open") async def on_open(): for symbol in SYMBOLS: - await bfx.wss.subscribe(Channel.BOOK, symbol=symbol, prec="R0") + await bfx.wss.subscribe("book", symbol=symbol, prec="R0") + @bfx.wss.on("subscribed") def on_subscribed(subscription): - print(f"Subscription successful for pair <{subscription['pair']}>") + print(f"Subscription successful for symbol <{subscription['symbol']}>") + @bfx.wss.on("t_raw_book_snapshot") def on_t_raw_book_snapshot(subscription: Book, snapshot: List[TradingPairRawBook]): for data in snapshot: raw_order_book.update(subscription["symbol"], data) + @bfx.wss.on("t_raw_book_update") def on_t_raw_book_update(subscription: Book, data: TradingPairRawBook): raw_order_book.update(subscription["symbol"], data) + @bfx.wss.on("checksum") async def on_checksum(subscription: Book, value: int): symbol = subscription["symbol"] @@ -95,11 +103,14 @@ async def on_checksum(subscription: Book, value: int): if raw_order_book.verify(symbol, value): raw_order_book.cooldown[symbol] = False elif not raw_order_book.cooldown[symbol]: - print("Mismatch between local and remote checksums: " - f"restarting book for symbol <{symbol}>...") + print( + "Mismatch between local and remote checksums: " + f"restarting book for symbol <{symbol}>..." + ) await bfx.wss.resubscribe(sub_id=subscription["sub_id"]) raw_order_book.cooldown[symbol] = True + bfx.wss.run() diff --git a/examples/websocket/public/ticker.py b/examples/websocket/public/ticker.py index 253d467..757ed2f 100644 --- a/examples/websocket/public/ticker.py +++ b/examples/websocket/public/ticker.py @@ -1,12 +1,11 @@ # python -c "import examples.websocket.public.ticker" -from bfxapi import Client, PUB_WSS_HOST - +from bfxapi import Client from bfxapi.types import TradingPairTicker from bfxapi.websocket.subscriptions import Ticker -from bfxapi.websocket.enums import Channel -bfx = Client(wss_host=PUB_WSS_HOST) +bfx = Client() + @bfx.wss.on("t_ticker_update") def on_t_ticker_update(subscription: Ticker, data: TradingPairTicker): @@ -14,8 +13,10 @@ def on_t_ticker_update(subscription: Ticker, data: TradingPairTicker): print(f"Data: {data}") + @bfx.wss.on("open") async def on_open(): - await bfx.wss.subscribe(Channel.TICKER, symbol="tBTCUSD") + await bfx.wss.subscribe("ticker", symbol="tBTCUSD") + bfx.wss.run() diff --git a/examples/websocket/public/trades.py b/examples/websocket/public/trades.py index 79dc71e..7dc6b07 100644 --- a/examples/websocket/public/trades.py +++ b/examples/websocket/public/trades.py @@ -1,25 +1,27 @@ # python -c "import examples.websocket.public.trades" -from bfxapi import Client, PUB_WSS_HOST - +from bfxapi import Client from bfxapi.types import Candle, TradingPairTrade from bfxapi.websocket.subscriptions import Candles, Trades -from bfxapi.websocket.enums import Error, Channel -bfx = Client(wss_host=PUB_WSS_HOST) +bfx = Client() + @bfx.wss.on("candles_update") def on_candles_update(_sub: Candles, candle: Candle): print(f"New candle: {candle}") + @bfx.wss.on("t_trade_execution") def on_t_trade_execution(_sub: Trades, trade: TradingPairTrade): print(f"New trade: {trade}") + @bfx.wss.on("open") async def on_open(): - await bfx.wss.subscribe(Channel.CANDLES, key="trade:1m:tBTCUSD") + await bfx.wss.subscribe("candles", key="trade:1m:tBTCUSD") + + await bfx.wss.subscribe("trades", symbol="tBTCUSD") - await bfx.wss.subscribe(Channel.TRADES, symbol="tBTCUSD") bfx.wss.run() From 5e50aa6f67e06f88cb8f378b59db334b690b55b3 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 18:04:00 +0200 Subject: [PATCH 62/65] Fix bug in BfxWebSocketClient::on's arguments (bfxapi/websocket/_client/bfx_websocket_client.py). --- bfxapi/websocket/_client/bfx_websocket_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bfxapi/websocket/_client/bfx_websocket_client.py b/bfxapi/websocket/_client/bfx_websocket_client.py index df6e8d5..e3f164f 100644 --- a/bfxapi/websocket/_client/bfx_websocket_client.py +++ b/bfxapi/websocket/_client/bfx_websocket_client.py @@ -336,5 +336,5 @@ class BfxWebSocketClient(Connection): await self._websocket.send(json.dumps( \ [ 0, event, None, data], cls=JSONEncoder)) - def on(self, event, f = None): - return self.__event_emitter.on(event, f) + def on(self, event, callback = None): + return self.__event_emitter.on(event, callback) From f3fe14b9216e1e278da0e4b1670fee0d8e46ed94 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 18:06:29 +0200 Subject: [PATCH 63/65] Add 'checksum' event in sub-package bfxapi.websocket._event_emitter. --- .../_event_emitter/bfx_event_emitter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py index 27f00d0..0ae50b4 100644 --- a/bfxapi/websocket/_event_emitter/bfx_event_emitter.py +++ b/bfxapi/websocket/_event_emitter/bfx_event_emitter.py @@ -28,15 +28,15 @@ _COMMON = [ "t_trade_execution", "t_trade_execution_update", "f_trade_execution", "f_trade_execution_update", "t_book_update", "f_book_update", "t_raw_book_update", "f_raw_book_update", "candles_update", - "derivatives_status_update", "liquidation_feed_update", "order_new", - "order_update", "order_cancel", "position_new", - "position_update", "position_close", "funding_offer_new", - "funding_offer_update", "funding_offer_cancel", "funding_credit_new", - "funding_credit_update", "funding_credit_close", "funding_loan_new", - "funding_loan_update", "funding_loan_close", "trade_execution", - "trade_execution_update", "wallet_update", "notification", - "on-req-notification", "ou-req-notification", "oc-req-notification", - "fon-req-notification", "foc-req-notification" + "derivatives_status_update", "liquidation_feed_update", "checksum", + "order_new", "order_update", "order_cancel", + "position_new", "position_update", "position_close", + "funding_offer_new", "funding_offer_update", "funding_offer_cancel", + "funding_credit_new", "funding_credit_update", "funding_credit_close", + "funding_loan_new", "funding_loan_update", "funding_loan_close", + "trade_execution", "trade_execution_update", "wallet_update", + "notification", "on-req-notification", "ou-req-notification", + "oc-req-notification", "fon-req-notification", "foc-req-notification" ] class BfxEventEmitter(AsyncIOEventEmitter): From afca5e306be0bd2e748f689dc987fda785248526 Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 18:07:18 +0200 Subject: [PATCH 64/65] Add support for Python 3.11 (edit setup.py). --- setup.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 9951f6e..4abcb5e 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ from distutils.core import setup -_version = {} -with open("bfxapi/_version.py", encoding="utf-8") as fp: - exec(fp.read(), _version) #pylint: disable=exec-used +_version = { } + +with open("bfxapi/_version.py", encoding="utf-8") as f: + #pylint: disable-next=exec-used + exec(f.read(), _version) setup( name="bitfinex-api-py", @@ -25,6 +27,7 @@ setup( "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], keywords="bitfinex,api,trading", project_urls={ @@ -32,10 +35,16 @@ setup( "Source": "https://github.com/bitfinexcom/bitfinex-api-py", }, packages=[ - "bfxapi", "bfxapi._utils", "bfxapi.types", - "bfxapi.websocket", "bfxapi.websocket._client", "bfxapi.websocket._handlers", - "bfxapi.websocket._event_emitter", - "bfxapi.rest", "bfxapi.rest.endpoints", "bfxapi.rest.middleware", + "bfxapi", + "bfxapi._utils", + "bfxapi.types", + "bfxapi.websocket", + "bfxapi.websocket._client", + "bfxapi.websocket._handlers", + "bfxapi.websocket._event_emitter", + "bfxapi.rest", + "bfxapi.rest.endpoints", + "bfxapi.rest.middleware", ], install_requires=[ "pyee~=9.0.4", @@ -44,4 +53,4 @@ setup( "urllib3~=1.26.14", ], python_requires=">=3.8" -) +) \ No newline at end of file From f63224c9052779639599d6c34bf00c69dcae3dff Mon Sep 17 00:00:00 2001 From: Davide Casale Date: Thu, 26 Oct 2023 18:10:15 +0200 Subject: [PATCH 65/65] Bump __version__ in file bfxapi/_version.py to v3.0.0b3. --- bfxapi/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bfxapi/_version.py b/bfxapi/_version.py index c9e4186..6ab976e 100644 --- a/bfxapi/_version.py +++ b/bfxapi/_version.py @@ -1 +1 @@ -__version__ = "3.0.0b2" +__version__ = "3.0.0b3"