diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index ea7405a..d7ea0c0 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -1,18 +1,16 @@ import os import pandas as pd import streamlit as st -import plotly.graph_objects as go import math -import plotly.express as px -from utils.os_utils import get_bots_data_paths +from utils.os_utils import get_databases from utils.database_manager import DatabaseManager -from utils.graphs import CandlesGraph -from utils.st_utils import initialize_st_page +from utils.graphs import PerformanceGraphs +from utils.st_utils import initialize_st_page, download_csv_button, style_metric_cards, db_error_message + initialize_st_page(title="Strategy Performance", icon="šŸš€") +style_metric_cards() -BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" -BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" UPLOAD_FOLDER = "data" # Start content here @@ -27,257 +25,10 @@ intervals = { "1d": 60 * 60 * 24, } - -def get_databases(): - databases = {} - bots_data_paths = get_bots_data_paths() - for source_name, source_path in bots_data_paths.items(): - sqlite_files = {} - for db_name in os.listdir(source_path): - if db_name.endswith(".sqlite"): - sqlite_files[db_name] = os.path.join(source_path, db_name) - databases[source_name] = sqlite_files - if len(databases) > 0: - return {key: value for key, value in databases.items() if value} - else: - return None - - -def download_csv(df: pd.DataFrame, filename: str, key: str): - csv = df.to_csv(index=False).encode('utf-8') - return st.download_button( - label="Press to Download", - data=csv, - file_name=f"{filename}.csv", - mime="text/csv", - key=key - ) - - -def style_metric_cards( - background_color: str = "rgba(255, 255, 255, 0)", - border_size_px: int = 1, - border_color: str = "rgba(255, 255, 255, 0.3)", - border_radius_px: int = 5, - border_left_color: str = "rgba(255, 255, 255, 0.5)", - box_shadow: bool = True, -): - - box_shadow_str = ( - "box-shadow: 0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15) !important;" - if box_shadow - else "box-shadow: none !important;" - ) - st.markdown( - f""" - - """, - unsafe_allow_html=True, - ) - - -def show_strategy_summary(summary_df: pd.DataFrame): - summary = st.data_editor(summary_df, - column_config={"PnL Over Time": st.column_config.LineChartColumn("PnL Over Time", - y_min=0, - y_max=5000), - "Explore": st.column_config.CheckboxColumn(required=True) - }, - use_container_width=True, - hide_index=True - ) - selected_rows = summary[summary.Explore] - if len(selected_rows) > 0: - return selected_rows - else: - return None - - -def summary_chart(df: pd.DataFrame): - fig = px.bar(df, x="Trading Pair", y="Realized PnL", color="Exchange") - fig.update_traces(width=min(1.0, 0.1 * len(strategy_data.strategy_summary))) - return fig - - -def pnl_over_time(df: pd.DataFrame): - df.reset_index(drop=True, inplace=True) - df_above = df[df['net_realized_pnl'] >= 0] - df_below = df[df['net_realized_pnl'] < 0] - - fig = go.Figure() - fig.add_trace(go.Bar(name="Cum Realized PnL", - x=df_above.index, - y=df_above["net_realized_pnl"], - marker_color=BULLISH_COLOR, - # hoverdq - showlegend=False)) - fig.add_trace(go.Bar(name="Cum Realized PnL", - x=df_below.index, - y=df_below["net_realized_pnl"], - marker_color=BEARISH_COLOR, - showlegend=False)) - fig.update_layout(title=dict( - text='Cummulative PnL', # Your title text - x=0.43, - y=0.95, - ), - plot_bgcolor='rgba(0,0,0,0)', - paper_bgcolor='rgba(0,0,0,0)') - return fig - - -def top_n_trades(series, n: int = 8): - podium = list(range(0, n)) - top_three_profits = series[series >= 0].sort_values(ascending=True)[-n:] - top_three_losses = series[series < 0].sort_values(ascending=False)[-n:] - fig = go.Figure() - fig.add_trace(go.Bar(name="Top Profits", - y=podium, - x=top_three_profits, - base=[0, 0, 0, 0], - marker_color=BULLISH_COLOR, - orientation='h', - text=top_three_profits.apply(lambda x: f"{x:.2f}"), - textposition="inside", - insidetextfont=dict(color='white'))) - fig.add_trace(go.Bar(name="Top Losses", - y=podium, - x=top_three_losses, - marker_color=BEARISH_COLOR, - orientation='h', - text=top_three_losses.apply(lambda x: f"{x:.2f}"), - textposition="inside", - insidetextfont=dict(color='white'))) - fig.update_layout(barmode='stack', - title=dict( - text='Top/Worst Realized PnLs', # Your title text - x=0.5, - y=0.95, - xanchor="center", - yanchor="top" - ), - xaxis=dict(showgrid=True, gridwidth=0.01, gridcolor="rgba(211, 211, 211, 0.5)"), # Show vertical grid lines - yaxis=dict(showgrid=False), - legend=dict(orientation="h", - x=0.5, - y=1.08, - xanchor="center", - yanchor="bottom")) - fig.update_yaxes(showticklabels=False, - showline=False, - range=[- n + 6, n + 1]) - return fig - - -def intraday_performance(df: pd.DataFrame): - def hr2angle(hr): - return (hr * 15) % 360 - - def hr_str(hr): - # Normalize hr to be between 1 and 12 - hr_str = str(((hr - 1) % 12) + 1) - suffix = ' AM' if (hr % 24) < 12 else ' PM' - return hr_str + suffix - - df["hour"] = df["timestamp"].dt.hour - realized_pnl_per_hour = df.groupby("hour")[["realized_pnl", "quote_volume"]].sum().reset_index() - fig = go.Figure() - fig.add_trace(go.Barpolar( - name="Profits", - r=realized_pnl_per_hour["quote_volume"], - theta=realized_pnl_per_hour["hour"] * 15, - marker=dict( - color=realized_pnl_per_hour["realized_pnl"], - colorscale="RdYlGn", - cmin=-(abs(realized_pnl_per_hour["realized_pnl"]).max()), - cmid=0.0, - cmax=(abs(realized_pnl_per_hour["realized_pnl"]).max()), - colorbar=dict( - title='Realized PnL', - x=0, - y=-0.5, - xanchor='left', - yanchor='bottom', - orientation='h' - ) - ))) - fig.update_layout( - polar=dict( - radialaxis=dict( - visible=True, - showline=False, - ), - angularaxis=dict( - rotation=90, - direction="clockwise", - tickvals=[hr2angle(hr) for hr in range(24)], - ticktext=[hr_str(hr) for hr in range(24)], - ), - bgcolor='rgba(255, 255, 255, 0)', - - ), - legend=dict( - orientation="h", - x=0.5, - y=1.08, - xanchor="center", - yanchor="bottom" - ), - title=dict( - text='Intraday Performance', - x=0.5, - y=0.93, - xanchor="center", - yanchor="bottom" - ), - ) - return fig - - -def returns_histogram(df: pd.DataFrame): - fig = go.Figure() - fig.add_trace(go.Histogram(name="Losses", - x=df.loc[df["realized_pnl"] < 0, "realized_pnl"], - marker_color=BEARISH_COLOR)) - fig.add_trace(go.Histogram(name="Profits", - x=df.loc[df["realized_pnl"] > 0, "realized_pnl"], - marker_color=BULLISH_COLOR)) - fig.update_layout( - title=dict( - text='Returns Distribution', - x=0.5, - xanchor="center", - ), - legend=dict( - orientation="h", - yanchor="bottom", - y=1.02, - xanchor="center", - x=.48 - )) - return fig - - -def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_rows=2): - cg = CandlesGraph(candles, show_volume=show_volume, extra_rows=extra_rows) - cg.add_buy_trades(strat_data.buys) - cg.add_sell_trades(strat_data.sells) - cg.add_pnl(strat_data, row=2) - cg.add_quote_inventory_change(strat_data, row=3) - return cg.figure() - - -style_metric_cards() +# Data source section st.subheader("šŸ”« Data source") + +# Upload database with st.expander("ā¬†ļø Upload"): uploaded_db = st.file_uploader("Select a Hummingbot SQLite Database", type=["sqlite", "db"]) if uploaded_db is not None: @@ -286,171 +37,200 @@ with st.expander("ā¬†ļø Upload"): f.write(file_contents) st.success("File uploaded and saved successfully!") selected_db = DatabaseManager(uploaded_db.name) + +# Find and select existing databases dbs = get_databases() if dbs is not None: bot_source = st.selectbox("Choose your database source:", dbs.keys()) db_names = [x for x in dbs[bot_source]] selected_db_name = st.selectbox("Select a database to start:", db_names) - executors_path = os.path.dirname(dbs[bot_source][selected_db_name]) - selected_db = DatabaseManager(db_name=dbs[bot_source][selected_db_name], - executors_path=executors_path) + selected_db = DatabaseManager(db_name=dbs[bot_source][selected_db_name]) else: st.warning("Ups! No databases were founded. Start uploading one") selected_db = None -if selected_db is not None: - strategy_data = selected_db.get_strategy_data() - if strategy_data.strategy_summary is not None: - st.divider() - st.subheader("šŸ“ Strategy summary") - table_tab, chart_tab = st.tabs(["Table", "Chart"]) - with table_tab: - selection = show_strategy_summary(strategy_data.strategy_summary) - if selection is not None: - if len(selection) > 1: - st.warning("This version doesn't support multiple selections. Please try selecting only one.") - st.stop() - selected_exchange = selection["Exchange"].values[0] - selected_trading_pair = selection["Trading Pair"].values[0] - with chart_tab: - summary_chart = summary_chart(strategy_data.strategy_summary) - st.plotly_chart(summary_chart, use_container_width=True) + st.stop() + +# Load strategy data +strategy_data = selected_db.get_strategy_data() +main_performance_charts = PerformanceGraphs(strategy_data) + +# Strategy summary section +st.divider() +st.subheader("šŸ“ Strategy summary") +if not main_performance_charts.has_summary_table: + db_error_message(db=selected_db, + error_message="Inaccesible summary table. Please try uploading a new database.") + st.stop() +else: + main_tab, chart_tab = st.tabs(["Main", "Chart"]) + with chart_tab: + st.plotly_chart(main_performance_charts.summary_chart(), use_container_width=True) + with main_tab: + selection = main_performance_charts.strategy_summary_table() if selection is None: st.info("šŸ’” Choose a trading pair and start analyzing!") + st.stop() + elif len(selection) > 1: + st.warning("This version doesn't support multiple selections. Please try selecting only one.") + st.stop() else: - st.divider() - st.subheader("šŸ” Explore Trading Pair") - if not any("Error" in value for key, value in selected_db.status.items() if key != "position_executor"): - date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) - start_time, end_time = st.select_slider("Select a time range to analyze", - options=date_array.tolist(), - value=(date_array[0], date_array[-1])) + selected_exchange = selection["Exchange"].values[0] + selected_trading_pair = selection["Trading Pair"].values[0] - single_market = True - if single_market: - single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) - strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) - with st.container(): - col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) - with col1: - st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.net_pnl_quote, 2)) - with col2: - st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) - with col3: - st.metric(label='Accuracy', - value=f"{100 * strategy_data_filtered.accuracy:.2f} %") - with col4: - st.metric(label="Profit Factor", - value=round(strategy_data_filtered.profit_factor, 2)) - with col5: - st.metric(label='Duration (Days)', - value=round(strategy_data_filtered.duration_seconds / (60 * 60 * 24), 2)) - with col6: - st.metric(label='Price change', - value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") - with col7: - buy_trades_amount = round(strategy_data_filtered.total_buy_amount, 2) - avg_buy_price = round(strategy_data_filtered.average_buy_price, 4) - st.metric(label="Total Buy Volume", - value=round(buy_trades_amount * avg_buy_price, 2)) - with col8: - sell_trades_amount = round(strategy_data_filtered.total_sell_amount, 2) - avg_sell_price = round(strategy_data_filtered.average_sell_price, 4) - st.metric(label="Total Sell Volume", - value=round(sell_trades_amount * avg_sell_price, 2)) - st.plotly_chart(pnl_over_time(strategy_data_filtered.trade_fill), use_container_width=True) - st.subheader("šŸ’± Market activity") - if "Error" not in selected_db.status["market_data"] and strategy_data_filtered.market_data is not None: - col1, col2, col3, col4 = st.columns(4) - with col1: - interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) - with col2: - rows_per_page = st.number_input("Candles per Page", value=1500, min_value=1, max_value=5000) - with col3: - st.markdown("##") - show_panel_metrics = st.checkbox("Show panel metrics", value=True) - with col4: - total_rows = len( - strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) - total_pages = math.ceil(total_rows / rows_per_page) - if total_pages > 1: - selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1, - key="page_slider") - else: - selected_page = 0 - start_idx = selected_page * rows_per_page - end_idx = start_idx + rows_per_page - candles_df = strategy_data_filtered.get_market_data_resampled( - interval=f"{intervals[interval]}S").iloc[ - start_idx:end_idx] - start_time_page = candles_df.index.min() - end_time_page = candles_df.index.max() - page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, - end_time_page) - if show_panel_metrics: - col1, col2 = st.columns([2, 1]) - with col1: - candles_chart = candles_graph(candles_df, page_data_filtered) - st.plotly_chart(candles_chart, use_container_width=True) - with col2: - chart_tab, table_tab = st.tabs(["Chart", "Table"]) - with chart_tab: - st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) - st.plotly_chart(returns_histogram(page_data_filtered.trade_fill), use_container_width=True) - with table_tab: - st.dataframe(page_data_filtered.trade_fill[["timestamp", "gross_pnl", "trade_fee", "realized_pnl"]].dropna(subset="realized_pnl"), - use_container_width=True, - hide_index=True, - height=(min(len(page_data_filtered.trade_fill) * 39, candles_chart.layout.height - 180))) - else: - st.plotly_chart(candles_graph(candles_df, page_data_filtered), use_container_width=True) - else: - st.warning("Market data is not available so the candles graph is not going to be rendered. " - "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") - st.divider() - st.subheader("šŸ“ˆ Metrics") - with st.container(): - col1, col2, col3, col4, col5 = st.columns(5) - with col1: - st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.trade_pnl_quote, 2)) - st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.cum_fees_in_quote, 2)) - with col2: - st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) - st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) - with col3: - st.metric(label='Total Buy Trades Amount', - value=round(strategy_data_filtered.total_buy_amount, 2)) - st.metric(label='Total Sell Trades Amount', - value=round(strategy_data_filtered.total_sell_amount, 2)) - with col4: - st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) - st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) - with col5: - st.metric(label='Inventory change in Base asset', - value=round(strategy_data_filtered.inventory_change_base_asset, 4)) - st.divider() - st.subheader("Tables") - with st.expander("šŸ’µ Trades"): - st.write(strategy_data.trade_fill) - download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") - with st.expander("šŸ“© Orders"): - st.write(strategy_data.orders) - download_csv(strategy_data.orders, "orders", "download-orders") - with st.expander("āŒ• Order Status"): - st.write(strategy_data.order_status) - download_csv(strategy_data.order_status, "order_status", "download-order-status") +# Explore Trading Pair section +st.divider() +st.subheader("šŸ” Explore Trading Pair") + +if any("Error" in status for status in [selected_db.status["trade_fill"], selected_db.status["orders"]]): + db_error_message(db=selected_db, + error_message="Database error. Check the status of your database.") + st.stop() + +# Filter strategy data by time +date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) +start_time, end_time = st.select_slider("Select a time range to analyze", + options=date_array.tolist(), + value=(date_array[0], date_array[-1])) +single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) +time_filtered_strategy_data = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) +time_filtered_performance_charts = PerformanceGraphs(time_filtered_strategy_data) + +# Header metrics +col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) +with col1: + st.metric(label=f'Net PNL {time_filtered_strategy_data.quote_asset}', + value=round(time_filtered_strategy_data.net_pnl_quote, 2), + help="The overall profit or loss achieved in quote asset.") +with col2: + st.metric(label='Total Trades', value=time_filtered_strategy_data.total_orders, + help="The total number of closed trades, winning and losing.") +with col3: + st.metric(label='Accuracy', + value=f"{100 * time_filtered_strategy_data.accuracy:.2f} %", + help="The percentage of winning trades, the number of winning trades divided by the total number of closed trades.") +with col4: + st.metric(label="Profit Factor", + value=round(time_filtered_strategy_data.profit_factor, 2), + help="The amount of money the strategy made for every unit of money it lost, net profits divided by gross losses.") +with col5: + st.metric(label='Duration (Days)', + value=round(time_filtered_strategy_data.duration_seconds / (60 * 60 * 24), 2), + help="The number of days the strategy was running.") +with col6: + st.metric(label='Price change', + value=f"{round(time_filtered_strategy_data.price_change * 100, 2)} %", + help="The percentage change in price from the start to the end of the strategy.") +with col7: + buy_trades_amount = round(time_filtered_strategy_data.total_buy_amount, 2) + avg_buy_price = round(time_filtered_strategy_data.average_buy_price, 4) + st.metric(label="Total Buy Volume", + value=round(buy_trades_amount * avg_buy_price, 2), + help="The total amount of quote asset bought.") +with col8: + sell_trades_amount = round(time_filtered_strategy_data.total_sell_amount, 2) + avg_sell_price = round(time_filtered_strategy_data.average_sell_price, 4) + st.metric(label="Total Sell Volume", + value=round(sell_trades_amount * avg_sell_price, 2), + help="The total amount of quote asset sold.") + +# Cummulative pnl chart +st.plotly_chart(time_filtered_performance_charts.pnl_over_time(), use_container_width=True) + +# Market activity section +st.subheader("šŸ’± Market activity") +if "Error" in selected_db.status["market_data"] or time_filtered_strategy_data.market_data.empty: + st.warning("Market data is not available so the candles graph is not going to be rendered." + "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") +else: + col1, col2 = st.columns([3, 1]) + with col2: + # Set custom configs + interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) + rows_per_page = st.number_input("Candles per Page", value=1500, min_value=1, max_value=5000) + + # Add pagination + total_rows = len(time_filtered_strategy_data.get_market_data_resampled(interval=f"{intervals[interval]}S")) + total_pages = math.ceil(total_rows / rows_per_page) + if total_pages > 1: + selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1, key="page_slider") + else: + selected_page = 0 + start_idx = selected_page * rows_per_page + end_idx = start_idx + rows_per_page + candles_df = time_filtered_strategy_data.get_market_data_resampled(interval=f"{intervals[interval]}S").iloc[start_idx:end_idx] + start_time_page = candles_df.index.min() + end_time_page = candles_df.index.max() + + # Get Page Filtered Strategy Data + page_filtered_strategy_data = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) + page_performance_charts = PerformanceGraphs(page_filtered_strategy_data) + candles_chart = page_performance_charts.candles_graph(candles_df, interval=interval) + + # Show auxiliary charts + intraday_tab, returns_tab, returns_data_tab, positions_tab, other_metrics_tab = st.tabs(["Intraday", "Returns", "Returns Data", "Positions", "Other Metrics"]) + with intraday_tab: + st.plotly_chart(time_filtered_performance_charts.intraday_performance(), use_container_width=True) + with returns_tab: + st.plotly_chart(time_filtered_performance_charts.returns_histogram(), use_container_width=True) + with returns_data_tab: + raw_returns_data = time_filtered_strategy_data.trade_fill[["timestamp", "gross_pnl", "trade_fee", "realized_pnl"]].dropna(subset="realized_pnl") + st.dataframe(raw_returns_data, + use_container_width=True, + hide_index=True, + height=(min(len(time_filtered_strategy_data.trade_fill) * 39, 600))) + download_csv_button(raw_returns_data, "raw_returns_data", "download-raw-returns") + with positions_tab: + positions_sunburst = page_performance_charts.position_executor_summary_sunburst() + if positions_sunburst: + st.plotly_chart(page_performance_charts.position_executor_summary_sunburst(), use_container_width=True) else: - st.warning("We are encountering challenges in maintaining continuous analysis of this database.") - with st.expander("DB Status"): - status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() - status_df.columns = ["Attribute", "Value"] - st.table(status_df) - else: - st.warning("We were unable to process this SQLite database.") - with st.expander("DB Status"): - status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() - status_df.columns = ["Attribute", "Value"] - st.table(status_df) + st.info("No position executor data found.") + with other_metrics_tab: + col3, col4 = st.columns(2) + with col3: + st.metric(label=f'Trade PNL {time_filtered_strategy_data.quote_asset}', + value=round(time_filtered_strategy_data.trade_pnl_quote, 2), + help="The overall profit or loss achieved in quote asset, without fees.") + st.metric(label='Total Buy Trades', value=time_filtered_strategy_data.total_buy_trades, + help="The total number of buy trades.") + st.metric(label='Total Buy Trades Amount', + value=round(time_filtered_strategy_data.total_buy_amount, 2), + help="The total amount of base asset bought.") + st.metric(label='Average Buy Price', value=round(time_filtered_strategy_data.average_buy_price, 4), + help="The average price of the base asset bought.") + + with col4: + st.metric(label=f'Fees {time_filtered_strategy_data.quote_asset}', + value=round(time_filtered_strategy_data.cum_fees_in_quote, 2), + help="The overall fees paid in quote asset.") + st.metric(label='Total Sell Trades', value=time_filtered_strategy_data.total_sell_trades, + help="The total number of sell trades.") + st.metric(label='Total Sell Trades Amount', + value=round(time_filtered_strategy_data.total_sell_amount, 2), + help="The total amount of base asset sold.") + st.metric(label='Average Sell Price', value=round(time_filtered_strategy_data.average_sell_price, 4), + help="The average price of the base asset sold.") + with col1: + st.plotly_chart(candles_chart, use_container_width=True) + +# Tables section +st.divider() +st.subheader("Tables") +with st.expander("šŸ’µ Trades"): + st.write(strategy_data.trade_fill) + download_csv_button(strategy_data.trade_fill, "trade_fill", "download-trades") +with st.expander("šŸ“© Orders"): + st.write(strategy_data.orders) + download_csv_button(strategy_data.orders, "orders", "download-orders") +with st.expander("āŒ• Order Status"): + st.write(strategy_data.order_status) + download_csv_button(strategy_data.order_status, "order_status", "download-order-status") +if not strategy_data.market_data.empty: + with st.expander("šŸ’± Market Data"): + st.write(strategy_data.market_data) + download_csv_button(strategy_data.market_data, "market_data", "download-market-data") +if strategy_data.position_executor is not None and not strategy_data.position_executor.empty: + with st.expander("šŸ¤– Position executor"): + st.write(strategy_data.position_executor) + download_csv_button(strategy_data.position_executor, "position_executor", "download-position-executor") diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index dcfc404..5fbaa01 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -1,6 +1,7 @@ import datetime from dataclasses import dataclass import pandas as pd +import numpy as np @dataclass @@ -19,28 +20,59 @@ class StrategyData: return None def get_strategy_summary(self): + columns_dict = {"strategy": "Strategy", + "market": "Exchange", + "symbol": "Trading Pair", + "order_id_count": "# Trades", + "total_positions": "# Positions", + "volume_sum": "Volume", + "TAKE_PROFIT": "# TP", + "STOP_LOSS": "# SL", + "TRAILING_STOP": "# TSL", + "TIME_LIMIT": "# TL", + "net_realized_pnl_full_series": "PnL Over Time", + "net_realized_pnl_last": "Realized PnL"} + def full_series(series): return list(series) - strategy_data = self.trade_fill.copy() - strategy_data["volume"] = strategy_data["amount"] * strategy_data["price"] - strategy_summary = strategy_data.groupby(["strategy", "market", "symbol"]).agg({"order_id": "count", - "volume": "sum", - "net_realized_pnl": [full_series, - "last"]}).reset_index() - strategy_summary.columns = [f"{col[0]}_{col[1]}" if isinstance(col, tuple) and col[1] is not None else col for col in strategy_summary.columns] - strategy_summary.rename(columns={"strategy_": "Strategy", - "market_": "Exchange", - "symbol_": "Trading Pair", - "order_id_count": "# Trades", - "volume_sum": "Volume", - "net_realized_pnl_full_series": "PnL Over Time", - "net_realized_pnl_last": "Realized PnL"}, inplace=True) + # Get trade fill data + trade_fill_data = self.trade_fill.copy() + trade_fill_data["volume"] = trade_fill_data["amount"] * trade_fill_data["price"] + grouped_trade_fill = trade_fill_data.groupby(["strategy", "market", "symbol"] + ).agg({"order_id": "count", + "volume": "sum", + "net_realized_pnl": [full_series, + "last"]}).reset_index() + grouped_trade_fill.columns = [f"{col[0]}_{col[1]}" if len(col[1]) > 0 else col[0] for col in grouped_trade_fill.columns] + + # Get position executor data + if self.position_executor is not None: + position_executor_data = self.position_executor.copy() + grouped_executors = position_executor_data.groupby(["exchange", "trading_pair", "controller_name", "close_type"]).agg(metric_count=("close_type", "count")).reset_index() + index_cols = ["exchange", "trading_pair", "controller_name"] + pivot_executors = pd.pivot_table(grouped_executors, values="metric_count", index=index_cols, columns="close_type").reset_index() + result_cols = ["TAKE_PROFIT", "STOP_LOSS", "TRAILING_STOP", "TIME_LIMIT"] + pivot_executors = pivot_executors.reindex(columns=index_cols + result_cols, fill_value=0) + pivot_executors["total_positions"] = pivot_executors[result_cols].sum(axis=1) + strategy_summary = grouped_trade_fill.merge(pivot_executors, left_on=["market", "symbol"], + right_on=["exchange", "trading_pair"], + how="left") + strategy_summary.drop(columns=["exchange", "trading_pair"], inplace=True) + else: + strategy_summary = grouped_trade_fill.copy() + strategy_summary["TAKE_PROFIT"] = np.nan + strategy_summary["STOP_LOSS"] = np.nan + strategy_summary["TRAILING_STOP"] = np.nan + strategy_summary["TIME_LIMIT"] = np.nan + strategy_summary["total_positions"] = np.nan + + strategy_summary.rename(columns=columns_dict, inplace=True) strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True) strategy_summary["Explore"] = False - column_names = list(strategy_summary.columns) - column_names.insert(0, column_names.pop()) - strategy_summary = strategy_summary[column_names] + sorted_cols = ["Explore", "Strategy", "Exchange", "Trading Pair", "# Trades", "Volume", "# Positions", + "# TP", "# SL", "# TSL", "# TL", "PnL Over Time", "Realized PnL"] + strategy_summary = strategy_summary.reindex(columns=sorted_cols, fill_value=0) return strategy_summary def get_single_market_strategy_data(self, exchange: str, trading_pair: str): diff --git a/utils/database_manager.py b/utils/database_manager.py index ccd237c..f6aa8b3 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -13,7 +13,6 @@ class DatabaseManager: self.db_name = db_name # TODO: Create db path for all types of db self.db_path = f'sqlite:///{os.path.join(db_name)}' - self.executors_path = executors_path self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False}) self.session_maker = sessionmaker(bind=self.engine) @@ -131,6 +130,18 @@ class DatabaseManager: query += f" WHERE {' AND '.join(conditions)}" return query + @staticmethod + def _get_position_executor_query(start_date=None, end_date=None): + query = "SELECT * FROM PositionExecutors" + conditions = [] + if start_date: + conditions.append(f"timestamp >= '{start_date}'") + if end_date: + conditions.append(f"timestamp <= '{end_date}'") + if conditions: + query += f" WHERE {' AND '.join(conditions)}" + return query + def get_orders(self, config_file_path=None, start_date=None, end_date=None): with self.session_maker() as session: query = self._get_orders_query(config_file_path, start_date, end_date) @@ -183,16 +194,10 @@ class DatabaseManager: return market_data def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataFrame: - df = pd.DataFrame() - files = [file for file in os.listdir(self.executors_path) if ".csv" in file and file != "trades_market_making_.csv"] - for file in files: - df0 = pd.read_csv(f"{self.executors_path}/{file}") - df = pd.concat([df, df0]) - df["datetime"] = pd.to_datetime(df["timestamp"], unit="s") - if start_date: - df = df[df["datetime"] >= start_date] - if end_date: - df = df[df["datetime"] <= end_date] - return df - - + with self.session_maker() as session: + query = self._get_position_executor_query(start_date, end_date) + position_executor = pd.read_sql_query(text(query), session.connection()) + position_executor.set_index("timestamp", inplace=True) + position_executor["datetime"] = pd.to_datetime(position_executor.index, unit="s") + position_executor["level"] = position_executor["order_level"].apply(lambda x: x.split("_")[1]) + return position_executor diff --git a/utils/graphs.py b/utils/graphs.py index f090978..608e1e0 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -1,7 +1,9 @@ import pandas as pd from plotly.subplots import make_subplots +import plotly.express as px import pandas_ta as ta # noqa: F401 import streamlit as st +from typing import Union from utils.data_manipulation import StrategyData, SingleMarketStrategyData from quants_lab.strategy.strategy_analysis import StrategyAnalysis @@ -10,11 +12,14 @@ import plotly.graph_objs as go BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" FEE_COLOR = "rgba(51, 0, 51, 0.9)" +MIN_INTERVAL_RESOLUTION = "1m" + class CandlesGraph: - def __init__(self, candles_df: pd.DataFrame, show_volume=True, extra_rows=1): + def __init__(self, candles_df: pd.DataFrame, line_mode=False, show_volume=True, extra_rows=1): self.candles_df = candles_df self.show_volume = show_volume + self.line_mode = line_mode rows, heights = self.get_n_rows_and_heights(extra_rows) self.rows = rows specs = [[{"secondary_y": True}]] * rows @@ -39,17 +44,37 @@ class CandlesGraph: return self.base_figure def add_candles_graph(self): - self.base_figure.add_trace( - go.Candlestick( - x=self.candles_df.index, - open=self.candles_df['open'], - high=self.candles_df['high'], - low=self.candles_df['low'], - close=self.candles_df['close'], - name="OHLC" - ), - row=1, col=1, - ) + if self.line_mode: + self.base_figure.add_trace( + go.Scatter(x=self.candles_df.index, + y=self.candles_df['close'], + name="Close", + mode='lines', + line=dict(color='blue')), + row=1, col=1, + ) + else: + hover_text = [] + for i in range(len(self.candles_df)): + hover_text.append( + f"Open: {self.candles_df['open'][i]}
" + f"High: {self.candles_df['high'][i]}
" + f"Low: {self.candles_df['low'][i]}
" + f"Close: {self.candles_df['close'][i]}
" + ) + self.base_figure.add_trace( + go.Candlestick( + x=self.candles_df.index, + open=self.candles_df['open'], + high=self.candles_df['high'], + low=self.candles_df['low'], + close=self.candles_df['close'], + name="OHLC", + hoverinfo="text", + hovertext=hover_text + ), + row=1, col=1, + ) def add_buy_trades(self, orders_data: pd.DataFrame): self.base_figure.add_trace( @@ -64,7 +89,9 @@ class CandlesGraph: size=12, line=dict(color='black', width=1), opacity=0.7, - )), + ), + hoverinfo="text", + hovertext=orders_data["price"].apply(lambda x: f"Buy Order: {x}
")), row=1, col=1, ) @@ -79,7 +106,9 @@ class CandlesGraph: color='red', size=12, line=dict(color='black', width=1), - opacity=0.7, )), + opacity=0.7,), + hoverinfo="text", + hovertext=orders_data["price"].apply(lambda x: f"Sell Order: {x}
")), row=1, col=1, ) @@ -206,6 +235,33 @@ class CandlesGraph: ) self.base_figure.update_yaxes(title_text='PNL', row=row, col=1) + def add_positions(self, position_executor_data: pd.DataFrame, row=1): + position_executor_data["close_datetime"] = pd.to_datetime(position_executor_data["close_timestamp"], unit="s") + i = 1 + for index, rown in position_executor_data.iterrows(): + i += 1 + self.base_figure.add_trace(go.Scatter(name=f"Position {index}", + x=[rown.datetime, rown.close_datetime], + y=[rown.entry_price, rown.close_price], + mode="lines", + line=dict(color="lightgreen" if rown.net_pnl_quote > 0 else "red"), + hoverinfo="text", + hovertext=f"Position NĀ°: {i}
" + f"Datetime: {rown.datetime}
" + f"Close datetime: {rown.close_datetime}
" + f"Side: {rown.side}
" + f"Entry price: {rown.entry_price}
" + f"Close price: {rown.close_price}
" + f"Close type: {rown.close_type}
" + f"Stop Loss: {100 * rown.sl:.2f}%
" + f"Take Profit: {100 * rown.tp:.2f}%
" + f"Time Limit: {100 * rown.tl:.2f}
" + f"Open Order Type: {rown.open_order_type}
" + f"Leverage: {rown.leverage}
" + f"Controller name: {rown.controller_name}
", + showlegend=False), + row=row, col=1) + def update_layout(self): self.base_figure.update_layout( title={ @@ -326,3 +382,204 @@ class BacktestingGraphs: strategy_analysis.create_base_figure(volume=add_volume, positions=add_positions, trade_pnl=add_pnl) st.plotly_chart(strategy_analysis.figure(), use_container_width=True) return metrics_container + + +class PerformanceGraphs: + BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" + BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" + FEE_COLOR = "rgba(51, 0, 51, 0.9)" + + def __init__(self, strategy_data: Union[StrategyData, SingleMarketStrategyData]): + self.strategy_data = strategy_data + + @property + def has_summary_table(self): + if isinstance(self.strategy_data, StrategyData): + return self.strategy_data.strategy_summary is not None + else: + return False + + @property + def has_position_executor_summary(self): + if isinstance(self.strategy_data, StrategyData): + return self.strategy_data.position_executor is not None + else: + return False + + def strategy_summary_table(self): + summary = st.data_editor(self.strategy_data.strategy_summary, + column_config={"PnL Over Time": st.column_config.LineChartColumn("PnL Over Time", + y_min=0, + y_max=5000), + "Explore": st.column_config.CheckboxColumn(required=True) + }, + use_container_width=True, + hide_index=True + ) + selected_rows = summary[summary.Explore] + if len(selected_rows) > 0: + return selected_rows + else: + return None + + def summary_chart(self): + fig = px.bar(self.strategy_data.strategy_summary, x="Trading Pair", y="Realized PnL", color="Exchange") + fig.update_traces(width=min(1.0, 0.1 * len(self.strategy_data.strategy_summary))) + return fig + + def pnl_over_time(self): + df = self.strategy_data.trade_fill.copy() + df.reset_index(drop=True, inplace=True) + df_above = df[df['net_realized_pnl'] >= 0] + df_below = df[df['net_realized_pnl'] < 0] + + fig = go.Figure() + fig.add_trace(go.Bar(name="Cum Realized PnL", + x=df_above.index, + y=df_above["net_realized_pnl"], + marker_color=BULLISH_COLOR, + # hoverdq + showlegend=False)) + fig.add_trace(go.Bar(name="Cum Realized PnL", + x=df_below.index, + y=df_below["net_realized_pnl"], + marker_color=BEARISH_COLOR, + showlegend=False)) + fig.update_layout(title=dict( + text='Cummulative PnL', # Your title text + x=0.43, + y=0.95, + ), + plot_bgcolor='rgba(0,0,0,0)', + paper_bgcolor='rgba(0,0,0,0)') + return fig + + def intraday_performance(self): + df = self.strategy_data.trade_fill.copy() + + def hr2angle(hr): + return (hr * 15) % 360 + + def hr_str(hr): + # Normalize hr to be between 1 and 12 + hr_string = str(((hr - 1) % 12) + 1) + suffix = ' AM' if (hr % 24) < 12 else ' PM' + return hr_string + suffix + + df["hour"] = df["timestamp"].dt.hour + realized_pnl_per_hour = df.groupby("hour")[["realized_pnl", "quote_volume"]].sum().reset_index() + fig = go.Figure() + fig.add_trace(go.Barpolar( + name="Profits", + r=realized_pnl_per_hour["quote_volume"], + theta=realized_pnl_per_hour["hour"] * 15, + marker=dict( + color=realized_pnl_per_hour["realized_pnl"], + colorscale="RdYlGn", + cmin=-(abs(realized_pnl_per_hour["realized_pnl"]).max()), + cmid=0.0, + cmax=(abs(realized_pnl_per_hour["realized_pnl"]).max()), + colorbar=dict( + title='Realized PnL', + x=0, + y=-0.5, + xanchor='left', + yanchor='bottom', + orientation='h' + ) + ))) + fig.update_layout( + polar=dict( + radialaxis=dict( + visible=True, + showline=False, + ), + angularaxis=dict( + rotation=90, + direction="clockwise", + tickvals=[hr2angle(hr) for hr in range(24)], + ticktext=[hr_str(hr) for hr in range(24)], + ), + bgcolor='rgba(255, 255, 255, 0)', + + ), + legend=dict( + orientation="h", + x=0.5, + y=1.08, + xanchor="center", + yanchor="bottom" + ), + title=dict( + text='Intraday Performance', + x=0.5, + y=0.93, + xanchor="center", + yanchor="bottom" + ), + ) + return fig + + def returns_histogram(self): + df = self.strategy_data.trade_fill.copy() + fig = go.Figure() + fig.add_trace(go.Histogram(name="Losses", + x=df.loc[df["realized_pnl"] < 0, "realized_pnl"], + marker_color=BEARISH_COLOR)) + fig.add_trace(go.Histogram(name="Profits", + x=df.loc[df["realized_pnl"] > 0, "realized_pnl"], + marker_color=BULLISH_COLOR)) + fig.update_layout( + title=dict( + text='Returns Distribution', + x=0.5, + xanchor="center", + ), + legend=dict( + orientation="h", + yanchor="bottom", + y=1.02, + xanchor="center", + x=.48 + )) + return fig + + def position_executor_summary_sunburst(self): + if self.strategy_data.position_executor is not None: + df = self.strategy_data.position_executor.copy() + grouped_df = df.groupby(["trading_pair", "side", "close_type"]).size().reset_index(name="count") + + fig = px.sunburst(grouped_df, + path=['trading_pair', 'side', 'close_type'], + values="count", + color_continuous_scale='RdBu', + color_continuous_midpoint=0) + + fig.update_layout( + title=dict( + text='Position Executor Summary', + x=0.5, + xanchor="center", + ), + legend=dict( + orientation="h", + yanchor="bottom", + y=1.02, + xanchor="center", + x=.48 + ) + ) + return fig + else: + return None + + def candles_graph(self, candles: pd.DataFrame, interval="5m", show_volume=False, extra_rows=2): + line_mode = interval == MIN_INTERVAL_RESOLUTION + cg = CandlesGraph(candles, show_volume=show_volume, line_mode=line_mode, extra_rows=extra_rows) + cg.add_buy_trades(self.strategy_data.buys) + cg.add_sell_trades(self.strategy_data.sells) + cg.add_pnl(self.strategy_data, row=2) + cg.add_quote_inventory_change(self.strategy_data, row=3) + if self.strategy_data.position_executor is not None: + cg.add_positions(self.strategy_data.position_executor, row=1) + return cg.figure() diff --git a/utils/os_utils.py b/utils/os_utils.py index c5cd6e2..5310e42 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -121,6 +121,21 @@ def get_bots_data_paths(): return data_sources +def get_databases(): + databases = {} + bots_data_paths = get_bots_data_paths() + for source_name, source_path in bots_data_paths.items(): + sqlite_files = {} + for db_name in os.listdir(source_path): + if db_name.endswith(".sqlite"): + sqlite_files[db_name] = os.path.join(source_path, db_name) + databases[source_name] = sqlite_files + if len(databases) > 0: + return {key: value for key, value in databases.items() if value} + else: + return None + + def get_function_from_file(file_path: str, function_name: str): # Create a module specification from the file path and load it spec = importlib.util.spec_from_file_location("module.name", file_path) diff --git a/utils/st_utils.py b/utils/st_utils.py index 189f07b..a89dcf5 100644 --- a/utils/st_utils.py +++ b/utils/st_utils.py @@ -1,10 +1,12 @@ import os.path +import pandas as pd from pathlib import Path import inspect import streamlit as st from st_pages import add_page_title +from utils.database_manager import DatabaseManager def initialize_st_page(title: str, icon: str, layout="wide", initial_sidebar_state="collapsed"): st.set_page_config( @@ -21,3 +23,56 @@ def initialize_st_page(title: str, icon: str, layout="wide", initial_sidebar_sta readme_path = current_directory / "README.md" with st.expander("About This Page"): st.write(readme_path.read_text()) + + +def download_csv_button(df: pd.DataFrame, filename: str, key: str): + csv = df.to_csv(index=False).encode('utf-8') + return st.download_button( + label="Download CSV", + data=csv, + file_name=f"{filename}.csv", + mime="text/csv", + key=key + ) + + +def style_metric_cards( + background_color: str = "rgba(255, 255, 255, 0)", + border_size_px: int = 1, + border_color: str = "rgba(255, 255, 255, 0.3)", + border_radius_px: int = 5, + border_left_color: str = "rgba(255, 255, 255, 0.5)", + box_shadow: bool = True, +): + + box_shadow_str = ( + "box-shadow: 0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15) !important;" + if box_shadow + else "box-shadow: none !important;" + ) + st.markdown( + f""" + + """, + unsafe_allow_html=True, + ) + + +def db_error_message(db: DatabaseManager, error_message: str): + container = st.container() + with container: + st.warning(error_message) + with st.expander("DB Status"): + status_df = pd.DataFrame([db.status]).transpose().reset_index() + status_df.columns = ["Attribute", "Value"] + st.table(status_df) + return container