(feat) add error handling for active instances

backend/services/backend_api_client.py

@@ -109,7 +109,9 @@ class BackendAPIClient:
         url = f"{self.base_url}/get-bot-status/{bot_name}"
         response = requests.get(url)
         if response.status_code == 200:
-            return response.json()["data"]
+            return response.json()
+        else:
+            return {"status": "error", "data": "Bot not found"}
 
     def get_bot_history(self, bot_name: str):
         """Get the historical data of a bot."""
@@ -127,7 +129,7 @@ class BackendAPIClient:
         if response.status_code == 200:
             return response.json()  # Successful request
         else:
-            return response.json()  # Handle errors or no data found
+            return {"status": "error", "data": "No active bots found"}
 
     def get_all_controllers_config(self):
         """Get all controller configurations."""

frontend/components/bot_performance_card.py

@@ -1,3 +1,5 @@
+import time
+
 from streamlit_elements import mui
 
 from CONFIG import BACKEND_API_HOST, BACKEND_API_PORT
@@ -44,10 +46,17 @@ class BotPerformanceCardV2(Dashboard.Item):
     def _handle_active_row_selection(self, params, _):
         self._active_controller_config_selected = params
 
-    def stop_controllers(self, bot_name):
+    def _handle_errors_row_selection(self, params, _):
+        self._error_controller_config_selected = params
+
+    def stop_active_controllers(self, bot_name):
         for controller in self._active_controller_config_selected:
             self._backend_api_client.stop_controller_from_bot(bot_name, controller)
 
+    def stop_errors_controllers(self, bot_name):
+        for controller in self._error_controller_config_selected:
+            self._backend_api_client.stop_controller_from_bot(bot_name, controller)
+
     def start_controllers(self, bot_name):
         for controller in self._stopped_controller_config_selected:
             self._backend_api_client.start_controller_from_bot(bot_name, controller)
@@ -56,26 +65,45 @@ class BotPerformanceCardV2(Dashboard.Item):
         try:
             controller_configs = backend_api_client.get_all_configs_from_bot(bot_name)
             bot_status = backend_api_client.get_bot_status(bot_name)
-            is_running = False
-            if bot_status and len(bot_status) > 0:
             # Controllers Table
             active_controllers_list = []
             stopped_controllers_list = []
+            error_controllers_list = []
             total_global_pnl_quote = 0
             total_volume_traded = 0
             total_open_order_volume = 0
             total_imbalance = 0
             total_unrealized_pnl_quote = 0
-                for controller, inner_dict in bot_status.items():
+            if bot_status.get("status") == "error":
-                    controller_config = next((config for config in controller_configs if config.get("id") == controller),
+                with mui.Card(key=self._key,
-                                             {})
+                              sx={"display": "flex", "flexDirection": "column", "borderRadius": 2, "overflow": "auto"},
+                              elevation=2):
+                    mui.CardHeader(
+                        title=bot_name,
+                        subheader="Not Available",
+                        avatar=mui.Avatar("🤖", sx={"bgcolor": "red"}),
+                        className=self._draggable_class)
+                    mui.Alert(f"An error occurred while fetching bot status of the bot {bot_name}. Please check the bot client.", severity="error")
+            else:
+                bot_data = bot_status.get("data")
+                is_running = bot_data.get("status") == "running"
+                performance = bot_data.get("performance")
+                if is_running:
+                    for controller, inner_dict in performance.items():
+                        controller_status = inner_dict.get("status")
+                        if controller_status == "error":
+                            error_controllers_list.append(
+                                {"id": controller, "error": inner_dict.get("error")})
+                            continue
+                        controller_performance = inner_dict.get("performance")
+                        controller_config = next((config for config in controller_configs if config.get("id") == controller), {})
                         kill_switch_status = True if controller_config.get("manual_kill_switch") is True else False
-                    realized_pnl_quote = inner_dict.get("realized_pnl_quote", 0)
+                        realized_pnl_quote = controller_performance.get("realized_pnl_quote", 0)
-                    unrealized_pnl_quote = inner_dict.get("unrealized_pnl_quote", 0)
+                        unrealized_pnl_quote = controller_performance.get("unrealized_pnl_quote", 0)
-                    global_pnl_quote = inner_dict.get("global_pnl_quote", 0)
+                        global_pnl_quote = controller_performance.get("global_pnl_quote", 0)
-                    volume_traded = inner_dict.get("volume_traded", 0)
+                        volume_traded = controller_performance.get("volume_traded", 0)
-                    open_order_volume = inner_dict.get("open_order_volume", 0)
+                        open_order_volume = controller_performance.get("open_order_volume", 0)
-                    imbalance = inner_dict.get("imbalance", 0)
+                        imbalance = controller_performance.get("imbalance", 0)
                         controller_info = {
                             "id": controller,
                             "realized_pnl_quote": realized_pnl_quote,
@@ -85,8 +113,6 @@ class BotPerformanceCardV2(Dashboard.Item):
                             "open_order_volume": open_order_volume,
                             "imbalance": imbalance,
                         }
-                    if len(controller_info) > 0:
-                        is_running = True
                         if kill_switch_status:
                             stopped_controllers_list.append(controller_info)
                         else:
@@ -124,7 +150,7 @@ class BotPerformanceCardV2(Dashboard.Item):
                                                    elevation=1):
                                         with self.title_bar(padding="10px 15px 10px 15px", dark_switcher=False):
                                             mui.Typography("🏦 NET PNL", variant="h6")
-                                    mui.Typography(f"$ {total_global_pnl_quote:.2f}", variant="h6", sx={"padding": "10px 15px 10px 15px"})
+                                        mui.Typography(f"$ {total_global_pnl_quote:.3f}", variant="h6", sx={"padding": "10px 15px 10px 15px"})
                                 with mui.Grid(item=True, xs=2):
                                     with mui.Paper(key=self._key,
                                                    sx={"display": "flex", "flexDirection": "column", "borderRadius": 3,
@@ -132,7 +158,7 @@ class BotPerformanceCardV2(Dashboard.Item):
                                                    elevation=1):
                                         with self.title_bar(padding="10px 15px 10px 15px", dark_switcher=False):
                                             mui.Typography("📊 NET PNL (%)", variant="h6")
-                                    mui.Typography(f"{total_global_pnl_pct:.2%}", variant="h6", sx={"padding": "10px 15px 10px 15px"})
+                                        mui.Typography(f"{total_global_pnl_pct:.3%}", variant="h6", sx={"padding": "10px 15px 10px 15px"})
                                 with mui.Grid(item=True, xs=2):
                                     with mui.Paper(key=self._key,
                                                    sx={"display": "flex", "flexDirection": "column", "borderRadius": 3,
@@ -185,12 +211,13 @@ class BotPerformanceCardV2(Dashboard.Item):
                                             hideFooter=True
                                         )
                                 with mui.Grid(item=True, xs=1):
-                                with mui.Button(onClick=lambda x: self.stop_controllers(bot_name),
+                                    with mui.Button(onClick=lambda x: self.stop_active_controllers(bot_name),
                                                     variant="outlined",
                                                     color="warning",
                                                     sx={"width": "100%", "height": "100%"}):
                                         mui.icon.AddCircleOutline()
                                         mui.Typography("Stop")
+                            if len(stopped_controllers_list) > 0:
                                 with mui.Grid(container=True, spacing=1, sx={"padding": "10px 15px 10px 15px"}):
                                     with mui.Grid(item=True, xs=11):
                                         with mui.Paper(key=self._key,
@@ -216,6 +243,32 @@ class BotPerformanceCardV2(Dashboard.Item):
                                                         sx={"width": "100%", "height": "100%"}):
                                             mui.icon.AddCircleOutline()
                                             mui.Typography("Start")
+                            if len(error_controllers_list) > 0:
+                                with mui.Grid(container=True, spacing=1, sx={"padding": "10px 15px 10px 15px"}):
+                                    with mui.Grid(item=True, xs=11):
+                                        with mui.Paper(key=self._key,
+                                                       sx={"display": "flex", "flexDirection": "column", "borderRadius": 3,
+                                                           "overflow": "hidden"},
+                                                       elevation=1):
+                                            with self.title_bar(padding="10px 15px 10px 15px", dark_switcher=False):
+                                                mui.Typography("💀 Controllers with errors", variant="h6")
+                                            mui.DataGrid(
+                                                rows=error_controllers_list,
+                                                columns=self.DEFAULT_COLUMNS,
+                                                autoHeight=True,
+                                                density="compact",
+                                                checkboxSelection=True,
+                                                disableSelectionOnClick=True,
+                                                onSelectionModelChange=self._handle_errors_row_selection,
+                                                hideFooter=True
+                                            )
+                                    with mui.Grid(item=True, xs=1):
+                                        with mui.Button(onClick=lambda x: self.stop_errors_controllers(bot_name),
+                                                        variant="outlined",
+                                                        color="warning",
+                                                        sx={"width": "100%", "height": "100%"}):
+                                            mui.icon.AddCircleOutline()
+                                            mui.Typography("Stop")
         except Exception as e:
             print(e)
             with mui.Card(key=self._key,

frontend/pages/orchestration/instances/app.py

@@ -21,7 +21,7 @@ def get_grid_positions(n_cards: int, cols: int = NUM_CARD_COLS, card_width: int
     return sorted(x_y, key=lambda x: (x[1], x[0]))
 
 
-def update_active_bots(api_client, active_instances_board):
+def update_active_bots(api_client):
     active_bots_response = api_client.get_active_bots_status()
     if active_bots_response.get("status") == "success":
         current_active_bots = active_bots_response.get("data")
@@ -29,16 +29,12 @@ def update_active_bots(api_client, active_instances_board):
 
         new_bots = set(current_active_bots.keys()) - set(stored_bots.keys())
         removed_bots = set(stored_bots.keys()) - set(current_active_bots.keys())
-
-        for bot in new_bots:
-            x, y = get_grid_positions(1)[0]  # Get a new position
-            card = BotPerformanceCardV2(active_instances_board, x, y, CARD_WIDTH, CARD_HEIGHT)
-            st.session_state.active_instances_board.bot_cards.append((card, bot))
-
         for bot in removed_bots:
             st.session_state.active_instances_board.bot_cards = [card for card in st.session_state.active_instances_board.bot_cards if card[1] != bot]
-
+        positions = get_grid_positions(len(current_active_bots), NUM_CARD_COLS, CARD_WIDTH, CARD_HEIGHT)
-        st.session_state.active_instances_board.bot_cards.sort(key=lambda x: x[1])  # Sort by bot name
+        for bot, (x, y) in zip(new_bots, positions[:len(new_bots)]):
+            card = BotPerformanceCardV2(st.session_state.active_instances_board.dashboard, x, y, CARD_WIDTH, CARD_HEIGHT)
+            st.session_state.active_instances_board.bot_cards.append((card, bot))
 
 
 initialize_st_page(title="Instances", icon="🦅")
@@ -49,26 +45,23 @@ if not api_client.is_docker_running():
     st.stop()
 
 if "active_instances_board" not in st.session_state:
-    try:
     active_bots_response = api_client.get_active_bots_status()
-        active_bots = active_bots_response.get("data")
     bot_cards = []
     board = Dashboard()
-        if active_bots:
-            positions = get_grid_positions(len(active_bots), NUM_CARD_COLS, CARD_WIDTH, CARD_HEIGHT)
-            for (bot, bot_info), (x, y) in zip(active_bots.items(), positions):
-                card = BotPerformanceCardV2(board, x, y, CARD_WIDTH, CARD_HEIGHT)
-                bot_cards.append((card, bot))
     st.session_state.active_instances_board = SimpleNamespace(
         dashboard=board,
         bot_cards=bot_cards,
     )
-    except Exception as e:
+    active_bots = active_bots_response.get("data")
-        st.error(f"Error fetching active bots, reload the page if persists: {e}")
+    number_of_bots = len(active_bots)
-        st.rerun()
+    if number_of_bots > 0:
+        positions = get_grid_positions(number_of_bots, NUM_CARD_COLS, CARD_WIDTH, CARD_HEIGHT)
+        for (bot, bot_info), (x, y) in zip(active_bots.items(), positions):
+            bot_status = api_client.get_bot_status(bot)
+            card = BotPerformanceCardV2(board, x, y, CARD_WIDTH, CARD_HEIGHT)
+            st.session_state.active_instances_board.bot_cards.append((card, bot))
 else:
-    active_instances_board = st.session_state.active_instances_board
+    update_active_bots(api_client)
-    update_active_bots(api_client, active_instances_board)
 
 with elements("active_instances_board"):
     with mui.Paper(sx={"padding": "2rem"}, variant="outlined"):
@@ -76,7 +69,3 @@ with elements("active_instances_board"):
         for card, bot in st.session_state.active_instances_board.bot_cards:
             with st.session_state.active_instances_board.dashboard():
                 card(bot)
-
-while True:
-    time.sleep(2)
-    st.rerun()