From 3c4b9cfd0dc4337131188e01e9f497bb46b7ae97 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 5 Sep 2023 23:39:11 -0300 Subject: [PATCH] (feat) improvements in status property and loading data in database_manager.py + including in strategy performance page --- pages/strategy_performance/app.py | 268 +++++++++++++++++------------- utils/database_manager.py | 78 ++++----- 2 files changed, 188 insertions(+), 158 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 813bbc0..c5fda1e 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -2,7 +2,7 @@ import os import pandas as pd import streamlit as st import math - +import plotly.express as px from utils.database_manager import DatabaseManager from utils.graphs import CandlesGraph from utils.st_utils import initialize_st_page @@ -10,6 +10,9 @@ from utils.st_utils import initialize_st_page initialize_st_page(title="Strategy Performance", icon="🚀") +BULLISH_COLOR = "#61C766" +BEARISH_COLOR = "#FF665A" + # Start content here intervals = { "1m": 60, @@ -41,126 +44,163 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): ) +def show_strategy_summary(summary_df: pd.DataFrame): + summary = st.data_editor(summary_df, + column_config={"PnL Over Time": st.column_config.LineChartColumn("PnL Over Time", + y_min=0, + y_max=5000), + "Examine": st.column_config.CheckboxColumn(required=True) + }, + use_container_width=True + ) + selected_rows = summary[summary.Examine] + return selected_rows.drop('Examine', axis=1) + + +def summary_chart(df: pd.DataFrame): + fig = px.bar(df, x="Trading Pair", y="Realized PnL", color="Exchange") + fig.update_traces(width=min(1.0, 0.1 * len(strategy_data.strategy_summary))) + return fig + + dbs = get_databases() -db_names = [x.db_name for x in dbs.values() if x.status == 'OK'] +db_names = [x.db_name for x in dbs.values()] if not db_names: st.warning("No trades have been recorded in the selected database") selected_db_name = None selected_db = None else: - st.subheader("⚙️ Filters") - col1, col2, col3, col4 = st.columns(4) - with col1: + st.subheader("🔫 Data source") + select_tab, upload_tab = st.tabs(["Select", "Upload"]) + with select_tab: selected_db_name = st.selectbox("Select a database to use:", db_names) selected_db = dbs[selected_db_name] - with col2: - if selected_db: - selected_config_file = st.selectbox("Select a config file to analyze:", selected_db.config_files) + with upload_tab: + uploaded_db = st.file_uploader("Upload your sqlite database", type=["sqlite", "db"]) + if uploaded_db is not None: + selected_db = DatabaseManager(uploaded_db) + strategy_data = selected_db.get_strategy_data() + if strategy_data.strategy_summary is not None: + st.subheader("📝 Strategy summary") + table_tab, chart_tab = st.tabs(["Table", "Chart"]) + with table_tab: + selection = show_strategy_summary(strategy_data.strategy_summary) + selected_exchange = selection["Exchange"].values[0] + selected_trading_pair = selection["Trading Pair"].values[0] + with chart_tab: + summary_chart = summary_chart(strategy_data.strategy_summary) + st.plotly_chart(summary_chart, use_container_width=True) + st.subheader("🔍 Examine Trading Pair") + if not any("🇽" in value for value in selected_db.status.values()): + date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) + start_time, end_time = st.select_slider("Select a time range to analyze", + options=date_array.tolist(), + value=(date_array[0], date_array[-1])) + + single_market = True + if single_market: + single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) + strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) + + st.divider() + with st.container(): + col1, col2 = st.columns(2) + with col1: + st.subheader(f"🏦 Market") + with col2: + st.subheader("📋 General stats") + col1, col2, col3, col4 = st.columns(4) + with col1: + st.metric(label="Exchange", value=strategy_data_filtered.exchange.capitalize()) + with col2: + st.metric(label="Trading pair", value=strategy_data_filtered.trading_pair.upper()) + with col3: + st.metric(label='Start date', value=strategy_data_filtered.start_time.strftime("%Y-%m-%d %H:%M")) + st.metric(label='End date', value=strategy_data_filtered.end_time.strftime("%Y-%m-%d %H:%M")) + with col4: + st.metric(label='Duration (Hours)', value=round(strategy_data_filtered.duration_seconds / 3600, 2)) + st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") + + st.divider() + st.subheader("📈 Performance") + col131, col132, col133, col134 = st.columns(4) + with col131: + st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.net_pnl_quote, 2)) + st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.trade_pnl_quote, 2)) + st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.cum_fees_in_quote, 2)) + with col132: + st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) + st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) + st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) + with col133: + st.metric(label='Inventory change in Base asset', + value=round(strategy_data_filtered.inventory_change_base_asset, 4)) + st.metric(label='Total Buy Trades Amount', + value=round(strategy_data_filtered.total_buy_amount, 2)) + st.metric(label='Total Sell Trades Amount', + value=round(strategy_data_filtered.total_sell_amount, 2)) + with col134: + st.metric(label='End Price', value=round(strategy_data_filtered.end_price, 4)) + st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) + st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) + + st.divider() + st.subheader("🕯️ Candlestick") + if strategy_data_filtered.market_data is not None: + with st.expander("Market activity", expanded=True): + col1, col2, col3 = st.columns([1, 1, 2]) + with col1: + interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) + with col2: + rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) + with col3: + total_rows = len(strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) + total_pages = math.ceil(total_rows / rows_per_page) + if total_pages > 1: + selected_page = st.select_slider("Select page", list(range(total_pages)), key="page_slider") + else: + selected_page = 0 + start_idx = selected_page * rows_per_page + end_idx = start_idx + rows_per_page + candles_df = strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S").iloc[ + start_idx:end_idx] + start_time_page = candles_df.index.min() + end_time_page = candles_df.index.max() + page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) + cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) + cg.add_buy_trades(strategy_data_filtered.buys) + cg.add_sell_trades(strategy_data_filtered.sells) + cg.add_pnl(strategy_data_filtered, row=2) + cg.add_base_inventory_change(strategy_data_filtered, row=3) + fig = cg.figure() + st.plotly_chart(fig, use_container_width=True) + else: + st.warning("Market data is not available so the candles graph is not going to be rendered. " + "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") + st.divider() + st.subheader("Tables") + with st.expander("💵 Trades"): + st.write(strategy_data.trade_fill) + download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") + with st.expander("📩 Orders"): + st.write(strategy_data.orders) + download_csv(strategy_data.orders, "orders", "download-orders") + with st.expander("⌕ Order Status"): + st.write(strategy_data.order_status) + download_csv(strategy_data.order_status, "order_status", "download-order-status") else: - selected_config_file = None - with col3: - if selected_config_file: - selected_exchange = st.selectbox("Exchange:", selected_db.configs[selected_config_file].keys()) - with col4: - if selected_exchange: - selected_trading_pair = st.selectbox("Trading Pair:", options=selected_db.configs[selected_config_file][selected_exchange]) + st.warning("We have problems to keep analyzing this database") + with st.expander("DB Status"): + status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() + status_df.columns = ["Attribute", "Value"] + st.table(status_df) - single_market = True - if single_market: - strategy_data = selected_db.get_strategy_data(selected_config_file) - single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) - date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) - start_time, end_time = st.select_slider("Select a time range to analyze", - options=date_array.tolist(), - value=(date_array[0], date_array[-1])) - strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) - - st.divider() - with st.container(): - col1, col2 = st.columns(2) - with col1: - st.subheader(f"🏦 Market") - with col2: - st.subheader("📋 General stats") - col1, col2, col3, col4 = st.columns(4) - with col1: - st.metric(label="Exchange", value=strategy_data_filtered.exchange.capitalize()) - with col2: - st.metric(label="Trading pair", value=strategy_data_filtered.trading_pair.upper()) - with col3: - st.metric(label='Start date', value=strategy_data_filtered.start_time.strftime("%Y-%m-%d %H:%M")) - st.metric(label='End date', value=strategy_data_filtered.end_time.strftime("%Y-%m-%d %H:%M")) - with col4: - st.metric(label='Duration (Hours)', value=round(strategy_data_filtered.duration_seconds / 3600, 2)) - st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") - - st.divider() - st.subheader("📈 Performance") - col131, col132, col133, col134 = st.columns(4) - with col131: - st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.net_pnl_quote, 2)) - st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.trade_pnl_quote, 2)) - st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.cum_fees_in_quote, 2)) - with col132: - st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) - st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) - st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) - with col133: - st.metric(label='Inventory change in Base asset', - value=round(strategy_data_filtered.inventory_change_base_asset, 4)) - st.metric(label='Total Buy Trades Amount', - value=round(strategy_data_filtered.total_buy_amount, 2)) - st.metric(label='Total Sell Trades Amount', - value=round(strategy_data_filtered.total_sell_amount, 2)) - with col134: - st.metric(label='End Price', value=round(strategy_data_filtered.end_price, 4)) - st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) - st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) - - st.divider() - st.subheader("🕯️ Candlestick") - if strategy_data_filtered.market_data is not None: - with st.expander("Market activity", expanded=True): - col1, col2, col3 = st.columns([1, 1, 2]) - with col1: - interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) - with col2: - rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) - with col3: - total_rows = len(strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) - total_pages = math.ceil(total_rows / rows_per_page) - if total_pages > 1: - selected_page = st.select_slider("Select page", list(range(total_pages)), key="page_slider") - else: - selected_page = 0 - start_idx = selected_page * rows_per_page - end_idx = start_idx + rows_per_page - candles_df = strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S").iloc[ - start_idx:end_idx] - start_time_page = candles_df.index.min() - end_time_page = candles_df.index.max() - page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) - cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) - cg.add_buy_trades(strategy_data_filtered.buys) - cg.add_sell_trades(strategy_data_filtered.sells) - cg.add_pnl(strategy_data_filtered, row=2) - cg.add_base_inventory_change(strategy_data_filtered, row=3) - fig = cg.figure() - st.plotly_chart(fig, use_container_width=True) - else: - st.warning("Market data is not available so the candles graph is not going to be rendered. " - "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") - st.divider() - st.subheader("Tables") - with st.expander("💵 Trades"): - st.write(strategy_data.trade_fill) - download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") - with st.expander("📩 Orders"): - st.write(strategy_data.orders) - download_csv(strategy_data.orders, "orders", "download-orders") - with st.expander("⌕ Order Status"): - st.write(strategy_data.order_status) - download_csv(strategy_data.order_status, "order_status", "download-order-status") + else: + st.warning("We couldn't process this sqlite database.") + with st.expander("DB Status"): + status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() + status_df.columns = ["Attribute", "Value"] + st.table(status_df) diff --git a/utils/database_manager.py b/utils/database_manager.py index 977c719..54db972 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -17,19 +17,41 @@ class DatabaseManager: self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False}) self.session_maker = sessionmaker(bind=self.engine) + def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None): + def load_data(table_loader): + try: + return table_loader() + except Exception as e: + return None # Return None to indicate failure + + # Use load_data to load tables + orders = load_data(self.get_orders) + trade_fills = load_data(self.get_trade_fills) + order_status = load_data(self.get_order_status) + market_data = load_data(self.get_market_data) + position_executor = load_data(self.get_position_executor_data) + + strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor) + return strategy_data + + @staticmethod + def _get_table_status(table_loader): + try: + data = table_loader() + return "Correct" if len(data) > 0 else f"Error - No records matched" + except Exception as e: + return f"Error - {str(e)}" + @property def status(self): - try: - with self.session_maker() as session: - query = 'SELECT DISTINCT config_file_path FROM TradeFill' - config_files = pd.read_sql_query(query, session.connection()) - if len(config_files) > 0: - # TODO: improve error handling, think what to do with other cases - return "OK" - else: - return "No records found in the TradeFill table with non-null config_file_path" - except Exception as e: - return f"Error: {str(e)}" + status = {"db_name": self.db_name, + "trade_fill": self._get_table_status(self.get_trade_fills), + "orders": self._get_table_status(self.get_orders), + "order_status": self._get_table_status(self.get_order_status), + "market_data": self._get_table_status(self.get_market_data), + "position_executor": self._get_table_status(self.get_position_executor_data), + } + return status @property def config_files(self): @@ -161,7 +183,7 @@ class DatabaseManager: def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataFrame: df = pd.DataFrame() - files = [file for file in os.listdir(self.executors_path) if ".csv" in file and file is not "trades_market_making_.csv"] + files = [file for file in os.listdir(self.executors_path) if ".csv" in file and file != "trades_market_making_.csv"] for file in files: df0 = pd.read_csv(f"{self.executors_path}/{file}") df = pd.concat([df, df0]) @@ -172,36 +194,4 @@ class DatabaseManager: df = df[df["datetime"] <= end_date] return df - @staticmethod - def _safe_table_loading(func, *args, **kwargs): - try: - table = func(*args, **kwargs) - except Exception: - table = None - return table - def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None): - def load_orders(): - return self.get_orders(config_file_path, start_date, end_date) - - def load_trade_fills(): - return self.get_trade_fills(config_file_path, start_date, end_date) - - def load_order_status(): - return self.get_order_status(orders['id'].tolist(), start_date, end_date) - - def load_market_data(): - return self.get_market_data(start_date, end_date) - - def load_position_executor(): - return self.get_position_executor_data(start_date, end_date) - - # Use _safe_table_loading to load tables - orders = self._safe_table_loading(load_orders) - trade_fills = self._safe_table_loading(load_trade_fills) - order_status = self._safe_table_loading(load_order_status) - market_data = self._safe_table_loading(load_market_data) - position_executor = self._safe_table_loading(load_position_executor) - - strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor) - return strategy_data