From 6947e6c0c58d29328d84e2caae9949fe85931118 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:41:09 +0100 Subject: [PATCH 01/64] (feat) add jupyter to dependencies --- environment_conda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment_conda.yml b/environment_conda.yml index e1ee4bc..1ee512f 100644 --- a/environment_conda.yml +++ b/environment_conda.yml @@ -17,5 +17,6 @@ dependencies: - pandas_ta - pyyaml - commlib-py + - jupyter - git+https://github.com/hummingbot/hbot-remote-client-py.git - git+https://github.com/hummingbot/docker-manager.git From 5e630d505e989cf557dda4597bb679b2a6ecf792 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:41:37 +0100 Subject: [PATCH 02/64] (feat) create quants-lab --- quants_lab/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 quants_lab/__init__.py diff --git a/quants_lab/__init__.py b/quants_lab/__init__.py new file mode 100644 index 0000000..e69de29 From 5b0843cc79a640bca32078d923f524fa9ec209a5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:41:47 +0100 Subject: [PATCH 03/64] (feat) remove ccxt --- pages/2_🚀_Strategy_Performance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pages/2_🚀_Strategy_Performance.py b/pages/2_🚀_Strategy_Performance.py index 72a09e5..963bc22 100644 --- a/pages/2_🚀_Strategy_Performance.py +++ b/pages/2_🚀_Strategy_Performance.py @@ -1,5 +1,4 @@ import os -import ccxt import pandas as pd import streamlit as st From aa1acef1ed7e3eadc92d299ef8c18620b9e6ee82 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:42:00 +0100 Subject: [PATCH 04/64] (feat) add directories --- quants_lab/backtesting/__init__.py | 0 quants_lab/labeling/__init__.py | 0 quants_lab/research_notebooks/__init__.py | 0 quants_lab/utils/__init__.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 quants_lab/backtesting/__init__.py create mode 100644 quants_lab/labeling/__init__.py create mode 100644 quants_lab/research_notebooks/__init__.py create mode 100644 quants_lab/utils/__init__.py diff --git a/quants_lab/backtesting/__init__.py b/quants_lab/backtesting/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quants_lab/labeling/__init__.py b/quants_lab/labeling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quants_lab/research_notebooks/__init__.py b/quants_lab/research_notebooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quants_lab/utils/__init__.py b/quants_lab/utils/__init__.py new file mode 100644 index 0000000..e69de29 From ec54ca589cc3e87a8a1ee3ba208a5ef2720a7b4d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:42:11 +0100 Subject: [PATCH 05/64] (feat) add triple barrier method --- quants_lab/labeling/triple_barrier_method.py | 84 ++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 quants_lab/labeling/triple_barrier_method.py diff --git a/quants_lab/labeling/triple_barrier_method.py b/quants_lab/labeling/triple_barrier_method.py new file mode 100644 index 0000000..d20c027 --- /dev/null +++ b/quants_lab/labeling/triple_barrier_method.py @@ -0,0 +1,84 @@ +from typing import Optional + +import numpy as np +import pandas as pd + + +def triple_barrier_method(df, tp=1.0, sl=1.0, tl=5, std_span: Optional[int] = 100, trade_cost=0.0006, max_executors: int = 1): + df.index = pd.to_datetime(df.timestamp, unit="ms") + if std_span: + df["target"] = df["close"].rolling(std_span).std() / df["close"] + else: + df["target"] = 1 / 100 + df["tl"] = df.index + pd.Timedelta(seconds=tl) + df.dropna(subset="target", inplace=True) + + df = apply_tp_sl_on_tl(df, tp=tp, sl=sl) + + df = get_bins(df, trade_cost) + + df['tp'] = df['close'] * (1 + df['target'] * tp * df["side"]) + df['sl'] = df['close'] * (1 - df['target'] * sl * df["side"]) + + df = add_active_signals(df, max_executors) + return df + + +def add_active_signals(df, max_executors): + close_times = [pd.Timestamp.min] * max_executors + df["active_signal"] = 0 + for index, row in df[(df["side"] != 0)].iterrows(): + for close_time in close_times: + if row["timestamp"] > close_time: + df.loc[df.index == index, "active_signal"] = 1 + close_times.remove(close_time) + close_times.append(row["close_time"]) + break + return df + + +def get_bins(df, trade_cost): + # 1) prices aligned with events + px = df.index.union(df['tl'].values).drop_duplicates() + px = df.close.reindex(px, method='ffill') + + # 2) create out object + df['ret'] = (px.loc[df['close_time'].values].values / px.loc[df.index] - 1) * df['side'] + df['real_class'] = np.sign(df['ret'] - trade_cost) + return df + + +def apply_tp_sl_on_tl(df: pd.DataFrame, tp: float, sl: float): + events = df[df["side"] != 0].copy() + if tp > 0: + take_profit = tp * events['target'] + else: + take_profit = pd.Series(index=df.index) # NaNs + if sl > 0: + stop_loss = - sl * events['target'] + else: + stop_loss = pd.Series(index=df.index) # NaNs + + for loc, tl in events['tl'].fillna(df.index[-1]).items(): + # In the future we can think about including High and Low prices in the calculation + # side = events.at[loc, 'side'] # side (1 or -1) + # sl = stop_loss[loc] + # tp = take_profit[loc] + # close = df.close[loc] # path close price + # # path_close = df.close[loc:tl] # path prices + # path_high = (df.high[loc:tl] / close) - 1 # path high prices + # path_low = (df.low[loc:tl] / close) - 1 # path low prices + # if side == 1: + # df.loc[loc, 'stop_loss_time'] = path_low[path_low < sl].index.min() # earliest stop loss. + # df.loc[loc, 'take_profit_time'] = path_high[path_high > tp].index.min() # earliest profit taking. + # elif side == -1: + # df.loc[loc, 'stop_loss_time'] = path_high[path_high > -sl].index.min() + # df.loc[loc, 'take_profit_time'] = path_low[path_low < -tp].index.min() + df0 = df.close[loc:tl] # path prices + df0 = (df0 / df.close[loc] - 1) * events.at[loc, 'side'] # path returns + df.loc[loc, 'stop_loss_time'] = df0[df0 < stop_loss[loc]].index.min() # earliest stop loss. + df.loc[loc, 'take_profit_time'] = df0[df0 > take_profit[loc]].index.min() # earliest profit taking. + df["close_time"] = df[["tl", "take_profit_time", "stop_loss_time"]].dropna(how='all').min(axis=1) + df['close_type'] = df[['take_profit_time', 'stop_loss_time', 'tl']].dropna(how='all').idxmin(axis=1) + df['close_type'].replace({'take_profit_time': 'tp', 'stop_loss_time': 'sl'}, inplace=True) + return df From 9f438190839be52ff7a80a951283b25cfad550bf Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:42:21 +0100 Subject: [PATCH 06/64] (feat) add function to get the candles --- quants_lab/utils/data_management.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 quants_lab/utils/data_management.py diff --git a/quants_lab/utils/data_management.py b/quants_lab/utils/data_management.py new file mode 100644 index 0000000..4160b9e --- /dev/null +++ b/quants_lab/utils/data_management.py @@ -0,0 +1,22 @@ +import os + +import pandas as pd + + +def get_dataframe(exchange: str, trading_pair: str, interval: str) -> pd.DataFrame: + """ + Get a dataframe of market data from the database. + :param exchange: Exchange name + :param trading_pair: Trading pair + :param interval: Interval of the data + :return: Dataframe of market data + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + data_dir = os.path.join(script_dir, "../../data/candles") + filename = f"candles_{exchange}_{trading_pair.upper()}_{interval}.csv" + file_path = os.path.join(data_dir, filename) + if not os.path.exists(file_path): + raise FileNotFoundError(f"File '{file_path}' does not exist.") + df = pd.read_csv(file_path) + df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms") + return df From 8cba46b036f6b5a92d599a09e32ba70f7718c71d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:42:36 +0100 Subject: [PATCH 07/64] (feat) add backtesting engine --- quants_lab/backtesting/backtesting.py | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 quants_lab/backtesting/backtesting.py diff --git a/quants_lab/backtesting/backtesting.py b/quants_lab/backtesting/backtesting.py new file mode 100644 index 0000000..b24874e --- /dev/null +++ b/quants_lab/backtesting/backtesting.py @@ -0,0 +1,60 @@ +from typing import Callable + +import pandas as pd + +from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis +from quants_lab.labeling.triple_barrier_method import triple_barrier_method + + +class Backtesting: + def __init__(self, candles_df): + self.candles_df = candles_df + + def run_backtesting(self, + strategy: Callable, + order_amount, leverage, initial_portfolio, + take_profit_multiplier, stop_loss_multiplier, time_limit, + std_span, taker_fee=0.0003, maker_fee=0.00012): + df = strategy(self.candles_df.copy()) + df = triple_barrier_method( + df=df, + std_span=std_span, + tp=take_profit_multiplier, + sl=stop_loss_multiplier, + tl=time_limit, + trade_cost=taker_fee * 2, + max_executors=1, + ) + + backtesting_analysis = self.get_backtest_analysis( + df=df, + maker_fee=maker_fee, + taker_fee=taker_fee, + order_amount=order_amount, + leverage=leverage, + initial_portfolio=initial_portfolio, + ) + return backtesting_analysis + + @staticmethod + def get_backtest_analysis( + df, + order_amount=15, + initial_portfolio=700, + leverage=20, + maker_fee=0.00012, + taker_fee=0.0003, + ): + # Set starting params + first_row = df.iloc[0].tolist() + first_row.extend([0, 0, 0, 0, 0, initial_portfolio]) + active_signals = df[df["active_signal"] == 1].copy() + active_signals.loc[:, "amount"] = order_amount + active_signals.loc[:, "margin_used"] = order_amount / leverage + active_signals.loc[:, "fee_pct"] = active_signals["close_type"].apply(lambda x: maker_fee + taker_fee if x == "tp" else taker_fee * 2) + active_signals.loc[:, "fee_usd"] = active_signals["fee_pct"] * active_signals["amount"] + active_signals.loc[:, "ret_usd"] = active_signals.apply(lambda x: (x["ret"] - x["fee_pct"]) * x["amount"], axis=1) + active_signals.loc[:, "current_portfolio"] = initial_portfolio + active_signals["ret_usd"].cumsum() + active_signals.loc[:, "current_portfolio"].fillna(method='ffill', inplace=True) + positions = pd.concat([pd.DataFrame([first_row], columns=active_signals.columns), active_signals]) + return positions.reset_index(drop=True) From 80f2be82b3c97c06c08c88db0d416b4bcbe5790d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:42:41 +0100 Subject: [PATCH 08/64] (feat) add backtesting analysis class --- .../backtesting/backtesting_analysis.py | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 quants_lab/backtesting/backtesting_analysis.py diff --git a/quants_lab/backtesting/backtesting_analysis.py b/quants_lab/backtesting/backtesting_analysis.py new file mode 100644 index 0000000..537aa65 --- /dev/null +++ b/quants_lab/backtesting/backtesting_analysis.py @@ -0,0 +1,197 @@ +import pandas as pd +from plotly.subplots import make_subplots +import pandas_ta as ta # noqa: F401 +import plotly.graph_objs as go +import numpy as np + + +class BacktestingAnalysis: + def __init__(self, candles_df: pd.DataFrame, positions: pd.DataFrame, show_volume=True, extra_rows=1): + self.candles_df = candles_df + self.positions = positions + self.show_volume = show_volume + rows, heights = self.get_n_rows_and_heights(extra_rows) + self.rows = rows + specs = [[{"secondary_y": True}]] * rows + self.base_figure = make_subplots(rows=rows, cols=1, shared_xaxes=True, vertical_spacing=0.05, + row_heights=heights, specs=specs) + self.add_candles_graph() + if self.show_volume: + self.add_volume() + self.update_layout() + + def add_positions(self): + # Add long and short positions + active_signals = self.positions.copy() + active_signals.loc[active_signals['side'] == -1, 'symbol'] = 'triangle-down' + active_signals.loc[active_signals['side'] == 1, 'symbol'] = 'triangle-up' + active_signals.loc[active_signals['real_class'] == 1, 'color'] = 'lightgreen' + active_signals.loc[active_signals['real_class'] == -1, 'color'] = 'red' + self.base_figure.add_trace(go.Scatter(x=active_signals.loc[(active_signals['side'] != 0), 'timestamp'], + y=active_signals.loc[active_signals['side'] != 0, 'close'], + name='Entry Price: $', + mode='markers', + marker_color=active_signals.loc[(active_signals['side'] != 0), 'color'], + marker_symbol=active_signals.loc[(active_signals['side'] != 0), 'symbol'], + marker_size=20, + marker_line={'color': 'black', 'width': 0.7})) + + for index, row in active_signals.iterrows(): + self.base_figure.add_shape(type="rect", + fillcolor="green", + opacity=0.5, + x0=row.timestamp, + y0=row.close, + x1=row.close_time, + y1=row.tp, + line=dict(color="green")) + # Add SL + self.base_figure.add_shape(type="rect", + fillcolor="red", + opacity=0.5, + x0=row.timestamp, + y0=row.close, + x1=row.close_time, + y1=row.sl, + line=dict(color="red")) + + def get_n_rows_and_heights(self, extra_rows): + rows = 1 + extra_rows + self.show_volume + row_heights = [0.5] * (extra_rows) + if self.show_volume: + row_heights.insert(0, 0.2) + row_heights.insert(0, 0.8) + return rows, row_heights + + def figure(self): + return self.base_figure + + def add_candles_graph(self): + self.base_figure.add_trace( + go.Candlestick( + x=self.candles_df['timestamp'], + open=self.candles_df['open'], + high=self.candles_df['high'], + low=self.candles_df['low'], + close=self.candles_df['close'], + name="OHLC" + ), + row=1, col=1, + ) + + def add_volume(self): + self.base_figure.add_trace( + go.Bar( + x=self.candles_df['timestamp'], + y=self.candles_df['volume'], + name="Volume", + opacity=0.5, + marker=dict(color='lightgreen') + ), + row=2, col=1, + ) + + def add_trade_pnl(self, row=4): + self.base_figure.add_trace( + go.Scatter( + x=self.positions['timestamp'], + y=self.positions['ret_usd'].cumsum(), + name="Cumulative Trade PnL", + mode='lines', + line=dict(color='chocolate', width=2)), + row=row, col=1 + ) + self.base_figure.update_yaxes(title_text='Cum Trade PnL', row=row, col=1) + + def update_layout(self): + self.base_figure.update_layout( + title={ + 'text': "Backtesting Analysis", + 'y': 0.95, + 'x': 0.5, + 'xanchor': 'center', + 'yanchor': 'top' + }, + legend=dict( + orientation="h", + yanchor="bottom", + y=-0.2, + xanchor="right", + x=1 + ), + height=1000, + xaxis_rangeslider_visible=False, + hovermode='x unified' + ) + self.base_figure.update_yaxes(title_text="Price", row=1, col=1) + if self.show_volume: + self.base_figure.update_yaxes(title_text="Volume", row=2, col=1) + self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) + + def initial_portfolio(self): + return self.positions['current_portfolio'].dropna().values[0] + + def final_portfolio(self): + return self.positions['current_portfolio'].dropna().values[-1] + + def net_profit_usd(self): + return self.final_portfolio() - self.initial_portfolio() + + def net_profit_pct(self): + return self.net_profit_usd() / self.initial_portfolio() + + def returns(self): + return self.positions['ret_usd'] / self.initial_portfolio() + + def total_positions(self): + return self.positions.shape[0] - 1 + + def win_signals(self): + return self.positions.loc[(self.positions['real_class'] > 0) & (self.positions["side"] != 0)] + + def loss_signals(self): + return self.positions.loc[(self.positions['real_class'] < 0) & (self.positions["side"] != 0)] + + def accuracy(self): + return self.win_signals().shape[0] / self.total_positions() + + def max_drawdown_usd(self): + cumulative_returns = self.positions["ret_usd"].cumsum() + peak = np.maximum.accumulate(cumulative_returns) + drawdown = (cumulative_returns - peak) + max_draw_down = np.min(drawdown) + return max_draw_down + + def max_drawdown_pct(self): + return self.max_drawdown_usd() / self.initial_portfolio() + + def sharpe_ratio(self): + returns = self.returns() + return returns.mean() / returns.std() + + def profit_factor(self): + total_won = self.win_signals().loc[:, 'ret_usd'].sum() + total_loss = - self.loss_signals().loc[:, 'ret_usd'].sum() + return total_won / total_loss + + def duration_in_minutes(self): + return (self.positions['timestamp'].iloc[-1] - self.positions['timestamp'].iloc[0]).total_seconds() / 60 + + def avg_trading_time_in_minutes(self): + time_diff_minutes = (pd.to_datetime(self.positions['close_time']) - self.positions['timestamp']).dt.total_seconds() / 60 + return time_diff_minutes.mean() + + def text_report(self): + return f""" +Strategy Performance Report: + - Net Profit: {self.net_profit_usd():,.2f} USD ({self.net_profit_pct() * 100:,.2f}%) + - Total Positions: {self.total_positions()} + - Win Signals: {self.win_signals().shape[0]} + - Loss Signals: {self.loss_signals().shape[0]} + - Accuracy: {self.accuracy():,.2f}% + - Profit Factor: {self.profit_factor():,.2f} + - Max Drawdown: {self.max_drawdown_usd():,.2f} USD | {self.max_drawdown_pct() * 100:,.2f}% + - Sharpe Ratio: {self.sharpe_ratio():,.2f} + - Duration: {self.duration_in_minutes() / 60:,.2f} Hours + - Average Trade Duration: {self.avg_trading_time_in_minutes():,.2f} minutes + """ From 54101ad171d6a3d663f729cf008b994be1cfd03e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:42:49 +0100 Subject: [PATCH 09/64] (feat) add naive backtesting page --- pages/9_⚙️_Backtesting.py | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 pages/9_⚙️_Backtesting.py diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py new file mode 100644 index 0000000..0fd02f5 --- /dev/null +++ b/pages/9_⚙️_Backtesting.py @@ -0,0 +1,61 @@ +import pandas as pd +import pandas_ta as ta +import streamlit as st + +from quants_lab.utils import data_management +from quants_lab.backtesting.backtesting import Backtesting +from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis + +st.set_page_config( + page_title="Hummingbot Dashboard", + page_icon="🚀", + layout="wide", + initial_sidebar_state="collapsed" +) +st.title("⚙️ Backtesting") + + +df = data_management.get_dataframe( + exchange='binance_perpetual', + trading_pair='BTC-USDT', + interval='1m', +) + +df_to_show = data_management.get_dataframe( + exchange='binance_perpetual', + trading_pair='BTC-USDT', + interval='1h', +) + +def bbands_strategy(df): + df.ta.bbands(length=100, std=3, append=True) + df["side"] = 0 + long_condition = df["BBP_100_3.0"] < -0.2 + short_condition = df["BBP_100_3.0"] > 1.2 + df.loc[long_condition, "side"] = 1 + df.loc[short_condition, "side"] = -1 + return df + +backtesting = Backtesting(candles_df=df) + +backtesting_result = backtesting.run_backtesting( + strategy=bbands_strategy, + order_amount=50, + leverage=20, + initial_portfolio=100, + take_profit_multiplier=1.5, + stop_loss_multiplier=0.8, + time_limit=60 * 60 * 24, + std_span=None, +) + + +backtesting_analysis = BacktestingAnalysis(df_to_show, backtesting_result, extra_rows=1, show_volume=False) +backtesting_analysis.add_trade_pnl(row=2) + +c1, c2 = st.columns([0.2, 0.8]) +with c1: + st.text(backtesting_analysis.text_report()) +with c2: + st.plotly_chart(backtesting_analysis.figure(), use_container_width=True) + From af7850dcfaca4d7a218bf0f671fc174b3405cd73 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:44:05 +0100 Subject: [PATCH 10/64] (feat) allow notebooks --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6f8c0a1..b50c1f1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,6 @@ __pycache__/ # C extensions *. -*.ipynb - secrets.toml .idea/* From b05ab6a76e4ca2cccbb97174981b0a612ace261c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:44:16 +0100 Subject: [PATCH 11/64] (feat) add mean reversion notebook --- .../mean_reversion_strat.ipynb | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 quants_lab/research_notebooks/mean_reversion_strat.ipynb diff --git a/quants_lab/research_notebooks/mean_reversion_strat.ipynb b/quants_lab/research_notebooks/mean_reversion_strat.ipynb new file mode 100644 index 0000000..e96348f --- /dev/null +++ b/quants_lab/research_notebooks/mean_reversion_strat.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2023-06-23T18:54:06.001204Z", + "start_time": "2023-06-23T18:54:05.418412Z" + } + }, + "outputs": [], + "source": [ + "import pandas_ta as ta\n", + "\n", + "from quants_lab.utils import data_management\n", + "from quants_lab.backtesting.backtesting import Backtesting\n", + "from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis\n", + "\n", + "df = data_management.get_dataframe(\n", + " exchange='binance_perpetual',\n", + " trading_pair='ETH-USDT',\n", + " interval='3m',\n", + ")\n", + "\n", + "def bbands_strategy(df):\n", + " df.ta.bbands(length=100, std=3, append=True)\n", + " df[\"side\"] = 0\n", + " long_condition = df[\"BBP_100_3.0\"] < 0.0\n", + " short_condition = df[\"BBP_100_3.0\"] > 1.0\n", + " df.loc[long_condition, \"side\"] = 1\n", + " df.loc[short_condition, \"side\"] = -1\n", + " return df" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "backtesting = Backtesting(candles_df=df)\n", + "\n", + "positions = backtesting.run_backtesting(\n", + " strategy=bbands_strategy,\n", + " order_amount=50,\n", + " leverage=20,\n", + " initial_portfolio=100,\n", + " take_profit_multiplier=0.5,\n", + " stop_loss_multiplier=5.0,\n", + " time_limit=60 * 60 * 1,\n", + " std_span=None,\n", + ")\n", + "backtesting_report = BacktestingAnalysis(df, positions, extra_rows=1, show_volume=False)\n" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-06-23T18:54:08.381976Z", + "start_time": "2023-06-23T18:54:06.002029Z" + } + } + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Strategy Performance Report:\n", + " - Net Profit: -4.07 USD (-4.07%)\n", + " - Total Positions: 978\n", + " - Win Signals: 581\n", + " - Loss Signals: 397\n", + " - Accuracy: 0.59%\n", + " - Profit Factor: 0.97\n", + " - Max Drawdown: -13.85 USD | -13.85%\n", + " - Sharpe Ratio: -0.01\n", + " - Duration: 6,478.55 Hours\n", + " - Average Trade Duration: 48.62 minutes\n", + " \n" + ] + } + ], + "source": [ + "print(backtesting_report.text_report())" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-06-23T18:54:09.658773Z", + "start_time": "2023-06-23T18:54:09.628910Z" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Set the backend to Plotly\n", + "pd.options.plotting.backend = \"plotly\"\n", + "\n", + "positions[\"ret_usd\"].cumsum().plot()" + ], + "metadata": { + "collapsed": false, + "is_executing": true, + "ExecuteTime": { + "start_time": "2023-06-23T18:54:11.277722Z" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 9dc7edb409077fc463a9c7751c7d0590f1d2a91c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 23 Jun 2023 20:44:57 +0100 Subject: [PATCH 12/64] (feat) add conf_client --- .../data_downloader/conf_client.yml | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 hummingbot_files/bot_configs/data_downloader/conf_client.yml diff --git a/hummingbot_files/bot_configs/data_downloader/conf_client.yml b/hummingbot_files/bot_configs/data_downloader/conf_client.yml new file mode 100644 index 0000000..5bc6b61 --- /dev/null +++ b/hummingbot_files/bot_configs/data_downloader/conf_client.yml @@ -0,0 +1,194 @@ +#################################### +### client_config_map config ### +#################################### + +instance_id: e90c0d6f2b1e2d54fa0c0a69612b07174320963b + +log_level: INFO + +debug_console: false + +strategy_report_interval: 900.0 + +logger_override_whitelist: +- hummingbot.strategy.arbitrage +- hummingbot.strategy.cross_exchange_market_making +- conf + +log_file_path: /home/hummingbot/logs + +kill_switch_mode: {} + +# What to auto-fill in the prompt after each import command (start/config) +autofill_import: disabled + +telegram_mode: {} + +# MQTT Bridge configuration. +mqtt_bridge: + mqtt_host: localhost + mqtt_port: 1883 + mqtt_username: '' + mqtt_password: '' + mqtt_namespace: hbot + mqtt_ssl: false + mqtt_logger: true + mqtt_notifier: true + mqtt_commands: true + mqtt_events: true + mqtt_external_events: true + mqtt_autostart: true + +# Error log sharing +send_error_logs: true + +# Can store the previous strategy ran for quick retrieval. +previous_strategy: null + +# Advanced database options, currently supports SQLAlchemy's included dialects +# Reference: https://docs.sqlalchemy.org/en/13/dialects/ +# To use an instance of SQLite DB the required configuration is +# db_engine: sqlite +# To use a DBMS the required configuration is +# db_host: 127.0.0.1 +# db_port: 3306 +# db_username: username +# db_password: password +# db_name: dbname +db_mode: + db_engine: sqlite + +pmm_script_mode: {} + +# Balance Limit Configurations +# e.g. Setting USDT and BTC limits on Binance. +# balance_asset_limit: +# binance: +# BTC: 0.1 +# USDT: 1000 +balance_asset_limit: + kucoin: {} + ciex: {} + ascend_ex_paper_trade: {} + crypto_com: {} + mock_paper_exchange: {} + btc_markets: {} + bitmart: {} + hitbtc: {} + loopring: {} + mexc: {} + polkadex: {} + bybit: {} + foxbit: {} + gate_io_paper_trade: {} + kucoin_paper_trade: {} + altmarkets: {} + ascend_ex: {} + bittrex: {} + probit_kr: {} + binance: {} + bybit_testnet: {} + okx: {} + bitmex: {} + binance_us: {} + probit: {} + gate_io: {} + lbank: {} + whitebit: {} + bitmex_testnet: {} + kraken: {} + huobi: {} + binance_paper_trade: {} + ndax_testnet: {} + coinbase_pro: {} + ndax: {} + bitfinex: {} + +# Fixed gas price (in Gwei) for Ethereum transactions +manual_gas_price: 50.0 + +# Gateway API Configurations +# default host to only use localhost +# Port need to match the final installation port for Gateway +gateway: + gateway_api_host: localhost + gateway_api_port: '15888' + +certs_path: /home/hummingbot/certs + +# Whether to enable aggregated order and trade data collection +anonymized_metrics_mode: + anonymized_metrics_interval_min: 15.0 + +# Command Shortcuts +# Define abbreviations for often used commands +# or batch grouped commands together +command_shortcuts: +- command: spreads + help: Set bid and ask spread + arguments: + - Bid Spread + - Ask Spread + output: + - config bid_spread $1 + - config ask_spread $2 + +# A source for rate oracle, currently ascend_ex, binance, coin_gecko, kucoin, gate_io +rate_oracle_source: + name: binance + +# A universal token which to display tokens values in, e.g. USD,EUR,BTC +global_token: + global_token_name: USD + global_token_symbol: $ + +# Percentage of API rate limits (on any exchange and any end point) allocated to this bot instance. +# Enter 50 to indicate 50%. E.g. if the API rate limit is 100 calls per second, and you allocate +# 50% to this setting, the bot will have a maximum (limit) of 50 calls per second +rate_limits_share_pct: 100.0 + +commands_timeout: + create_command_timeout: 10.0 + other_commands_timeout: 30.0 + +# Tabulate table format style (https://github.com/astanin/python-tabulate#table-format) +tables_format: psql + +paper_trade: + paper_trade_exchanges: + - binance + - kucoin + - ascend_ex + - gate_io + paper_trade_account_balance: + BTC: 1.0 + USDT: 1000.0 + ONE: 1000.0 + USDQ: 1000.0 + TUSD: 1000.0 + ETH: 10.0 + WETH: 10.0 + USDC: 1000.0 + DAI: 1000.0 + +color: + top_pane: '#000000' + bottom_pane: '#000000' + output_pane: '#262626' + input_pane: '#1C1C1C' + logs_pane: '#121212' + terminal_primary: '#5FFFD7' + primary_label: '#5FFFD7' + secondary_label: '#FFFFFF' + success_label: '#5FFFD7' + warning_label: '#FFFF00' + info_label: '#5FD7FF' + error_label: '#FF0000' + gold_label: '#FFD700' + silver_label: '#C0C0C0' + bronze_label: '#CD7F32' + +# The tick size is the frequency with which the clock notifies the time iterators by calling the +# c_tick() method, that means for example that if the tick size is 1, the logic of the strategy +# will run every second. +tick_size: 1.0 From 5bb8fb25f7cfa561fd2c7266d2fae3db2e8404e4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 28 Jun 2023 17:52:01 +0100 Subject: [PATCH 13/64] (feat) add base class to create strategies and use graphs to show them --- .../strategy/directional_strategy_base.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 quants_lab/strategy/directional_strategy_base.py diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py new file mode 100644 index 0000000..d878df8 --- /dev/null +++ b/quants_lab/strategy/directional_strategy_base.py @@ -0,0 +1,31 @@ +from datetime import datetime +from typing import Optional + + +class DirectionalStrategyBase: + + def get_data(self, start: Optional[str] = None, end: Optional[str] = None): + df = self.get_raw_data() + return self.filter_df_by_time(df, start, end) + + def get_raw_data(self): + raise NotImplemented + + def add_indicators(self, df): + raise NotImplemented + + def add_signals(self, df): + raise NotImplemented + + @staticmethod + def filter_df_by_time(df, start: Optional[str] = None, end: Optional[str] = None): + timeframe_conditions = [] + if start is not None: + timeframe_conditions.append(df["timestamp"] >= datetime.strptime(start, "%Y-%m-%d")) + if end is not None: + timeframe_conditions.append(df["timestamp"] <= datetime.strptime(end, "%Y-%m-%d")) + if len(timeframe_conditions) > 0: + df = df.loc[timeframe_conditions[0] & timeframe_conditions[1]] + else: + df = df.copy() + return df \ No newline at end of file From ed3188b7989a34eced1874fc971dcebf277b90e9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 28 Jun 2023 17:52:19 +0100 Subject: [PATCH 14/64] (feat) use the official hummingbot dev version --- hummingbot_files/compose_files/data-downloader-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot_files/compose_files/data-downloader-compose.yml b/hummingbot_files/compose_files/data-downloader-compose.yml index 56af64b..0f8409e 100644 --- a/hummingbot_files/compose_files/data-downloader-compose.yml +++ b/hummingbot_files/compose_files/data-downloader-compose.yml @@ -2,7 +2,7 @@ version: "3.9" services: bot: container_name: candles_downloader - image: dardonacci/hummingbot:development + image: hummingbot/hummingbot:development volumes: - "../../data/candles:/home/hummingbot/data" - "../bot_configs/data_downloader/conf:/home/hummingbot/conf" From edf6caf4433ee2542854e87e3bdb5aab826cd908 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 28 Jun 2023 17:52:35 +0100 Subject: [PATCH 15/64] (feat) add to the conf file the market recording params --- .../bot_configs/data_downloader/conf/conf_client.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hummingbot_files/bot_configs/data_downloader/conf/conf_client.yml b/hummingbot_files/bot_configs/data_downloader/conf/conf_client.yml index 9f76adf..f6ec342 100644 --- a/hummingbot_files/bot_configs/data_downloader/conf/conf_client.yml +++ b/hummingbot_files/bot_configs/data_downloader/conf/conf_client.yml @@ -191,4 +191,9 @@ color: # The tick size is the frequency with which the clock notifies the time iterators by calling the # c_tick() method, that means for example that if the tick size is 1, the logic of the strategy # will run every second. -tick_size: 1.0 \ No newline at end of file +tick_size: 1.0 + +market_data_collection: + market_data_collection_enabled: true + market_data_collection_interval: 60 + market_data_collection_depth: 20 From 5804837c856d9632c9caa7ddd16d8123fcd236bb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 28 Jun 2023 17:52:53 +0100 Subject: [PATCH 16/64] (feat) create the first strategy example --- quants_lab/strategy/__init__.py | 0 .../strategy/mean_reversion/__init__.py | 0 .../strategy/mean_reversion/bollinger.py | 38 +++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 quants_lab/strategy/__init__.py create mode 100644 quants_lab/strategy/mean_reversion/__init__.py create mode 100644 quants_lab/strategy/mean_reversion/bollinger.py diff --git a/quants_lab/strategy/__init__.py b/quants_lab/strategy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quants_lab/strategy/mean_reversion/__init__.py b/quants_lab/strategy/mean_reversion/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quants_lab/strategy/mean_reversion/bollinger.py b/quants_lab/strategy/mean_reversion/bollinger.py new file mode 100644 index 0000000..adc5318 --- /dev/null +++ b/quants_lab/strategy/mean_reversion/bollinger.py @@ -0,0 +1,38 @@ +import pandas_ta as ta +from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase + +from quants_lab.utils import data_management + + +class Bollinger(DirectionalStrategyBase): + def __init__(self, + exchange="binance_perpetual", + trading_pair="ETH-USDT", + interval="1h", + bb_length=24, + bb_std=2.0): + self.exchange = exchange + self.trading_pair = trading_pair + self.interval = interval + self.bb_length = bb_length + self.bb_std = bb_std + + def get_raw_data(self): + df = data_management.get_dataframe( + exchange=self.exchange, + trading_pair=self.trading_pair, + interval=self.interval, + ) + return df + + def add_indicators(self, df): + df.ta.bbands(length=self.bb_length, std=self.bb_std, append=True) + return df + + def add_signals(self, df): + df["side"] = 0 + long_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] < 0.05 + short_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] > 0.95 + df.loc[long_condition, "side"] = 1 + df.loc[short_condition, "side"] = -1 + return df From 3fbda91b5c187e99e3ca859382978027eceb6154 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 20:37:09 +0100 Subject: [PATCH 17/64] (feat) refactor backtesting page --- quants_lab/backtesting/backtesting.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/quants_lab/backtesting/backtesting.py b/quants_lab/backtesting/backtesting.py index b24874e..cedc75f 100644 --- a/quants_lab/backtesting/backtesting.py +++ b/quants_lab/backtesting/backtesting.py @@ -1,21 +1,24 @@ -from typing import Callable +from typing import Optional import pandas as pd from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis from quants_lab.labeling.triple_barrier_method import triple_barrier_method +from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase class Backtesting: - def __init__(self, candles_df): - self.candles_df = candles_df + def __init__(self, strategy: DirectionalStrategyBase): + self.strategy = strategy def run_backtesting(self, - strategy: Callable, order_amount, leverage, initial_portfolio, take_profit_multiplier, stop_loss_multiplier, time_limit, - std_span, taker_fee=0.0003, maker_fee=0.00012): - df = strategy(self.candles_df.copy()) + std_span, taker_fee=0.0003, maker_fee=0.00012, + start: Optional[str] = None, end: Optional[str] = None): + df = self.strategy.get_data(start=start, end=end) + df = self.strategy.add_indicators(df) + df = self.strategy.add_signals(df) df = triple_barrier_method( df=df, std_span=std_span, From 80fd5ee7137717e113d4fdba27de4934a1c69d98 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 20:37:17 +0100 Subject: [PATCH 18/64] (feat) refactor backtesting analysis --- .../backtesting/backtesting_analysis.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/quants_lab/backtesting/backtesting_analysis.py b/quants_lab/backtesting/backtesting_analysis.py index 537aa65..35baa61 100644 --- a/quants_lab/backtesting/backtesting_analysis.py +++ b/quants_lab/backtesting/backtesting_analysis.py @@ -1,3 +1,5 @@ +from typing import Optional + import pandas as pd from plotly.subplots import make_subplots import pandas_ta as ta # noqa: F401 @@ -6,19 +8,21 @@ import numpy as np class BacktestingAnalysis: - def __init__(self, candles_df: pd.DataFrame, positions: pd.DataFrame, show_volume=True, extra_rows=1): + def __init__(self, positions: pd.DataFrame, candles_df: Optional[pd.DataFrame] = None): self.candles_df = candles_df self.positions = positions - self.show_volume = show_volume - rows, heights = self.get_n_rows_and_heights(extra_rows) + + def create_base_figure(self, candlestick=True, volume=True, extra_rows=1): + rows, heights = self.get_n_rows_and_heights(extra_rows, volume) self.rows = rows specs = [[{"secondary_y": True}]] * rows self.base_figure = make_subplots(rows=rows, cols=1, shared_xaxes=True, vertical_spacing=0.05, row_heights=heights, specs=specs) - self.add_candles_graph() - if self.show_volume: + if candlestick: + self.add_candles_graph() + if volume: self.add_volume() - self.update_layout() + self.update_layout(volume) def add_positions(self): # Add long and short positions @@ -55,10 +59,10 @@ class BacktestingAnalysis: y1=row.sl, line=dict(color="red")) - def get_n_rows_and_heights(self, extra_rows): - rows = 1 + extra_rows + self.show_volume + def get_n_rows_and_heights(self, extra_rows, volume=True): + rows = 1 + extra_rows + volume row_heights = [0.5] * (extra_rows) - if self.show_volume: + if volume: row_heights.insert(0, 0.2) row_heights.insert(0, 0.8) return rows, row_heights @@ -103,7 +107,7 @@ class BacktestingAnalysis: ) self.base_figure.update_yaxes(title_text='Cum Trade PnL', row=row, col=1) - def update_layout(self): + def update_layout(self, volume=True): self.base_figure.update_layout( title={ 'text': "Backtesting Analysis", @@ -124,7 +128,7 @@ class BacktestingAnalysis: hovermode='x unified' ) self.base_figure.update_yaxes(title_text="Price", row=1, col=1) - if self.show_volume: + if volume: self.base_figure.update_yaxes(title_text="Volume", row=2, col=1) self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) From a0eb7e626935f2905324fe6b7e8cc2ffe41f561a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 20:37:30 +0100 Subject: [PATCH 19/64] (feat) add bollinger strategy --- quants_lab/strategy/mean_reversion/bollinger.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/quants_lab/strategy/mean_reversion/bollinger.py b/quants_lab/strategy/mean_reversion/bollinger.py index adc5318..9df5aaa 100644 --- a/quants_lab/strategy/mean_reversion/bollinger.py +++ b/quants_lab/strategy/mean_reversion/bollinger.py @@ -10,12 +10,16 @@ class Bollinger(DirectionalStrategyBase): trading_pair="ETH-USDT", interval="1h", bb_length=24, - bb_std=2.0): + bb_std=2.0, + bb_long_threshold=0.0, + bb_short_threshold=1.0,): self.exchange = exchange self.trading_pair = trading_pair self.interval = interval self.bb_length = bb_length self.bb_std = bb_std + self.bb_long_threshold = bb_long_threshold + self.bb_short_threshold = bb_short_threshold def get_raw_data(self): df = data_management.get_dataframe( @@ -31,7 +35,7 @@ class Bollinger(DirectionalStrategyBase): def add_signals(self, df): df["side"] = 0 - long_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] < 0.05 + long_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] < self.bb_long_threshold short_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] > 0.95 df.loc[long_condition, "side"] = 1 df.loc[short_condition, "side"] = -1 From 893a6ba4aba19188472b9bb9ee5616d62cf99c99 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 20:37:37 +0100 Subject: [PATCH 20/64] (feat) add macd_bb strategy --- quants_lab/strategy/mean_reversion/macd_bb.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 quants_lab/strategy/mean_reversion/macd_bb.py diff --git a/quants_lab/strategy/mean_reversion/macd_bb.py b/quants_lab/strategy/mean_reversion/macd_bb.py new file mode 100644 index 0000000..fb691b9 --- /dev/null +++ b/quants_lab/strategy/mean_reversion/macd_bb.py @@ -0,0 +1,54 @@ +import pandas_ta as ta +from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase + +from quants_lab.utils import data_management + + +class MACDBB(DirectionalStrategyBase): + def __init__(self, + exchange="binance_perpetual", + trading_pair="ETH-USDT", + interval="1h", + bb_length=24, + bb_std=2.0, + bb_long_threshold=0.05, + bb_short_threshold=0.95, + fast_macd=21, + slow_macd=42, + signal_macd=9): + self.exchange = exchange + self.trading_pair = trading_pair + self.interval = interval + self.bb_length = bb_length + self.bb_std = bb_std + self.bb_long_threshold = bb_long_threshold + self.bb_short_threshold = bb_short_threshold + self.fast_macd = fast_macd + self.slow_macd = slow_macd + self.signal_macd = signal_macd + + def get_raw_data(self): + df = data_management.get_dataframe( + exchange=self.exchange, + trading_pair=self.trading_pair, + interval=self.interval, + ) + return df + + def add_indicators(self, df): + df.ta.bbands(length=self.bb_length, std=self.bb_std, append=True) + df.ta.macd(fast=self.fast_macd, slow=self.slow_macd, signal=self.signal_macd, append=True) + return df + + def add_signals(self, df): + bbp = df[f"BBP_{self.bb_length}_{self.bb_std}"] + macdh = df[f"MACDh_{self.fast_macd}_{self.slow_macd}_{self.signal_macd}"] + macd = df[f"MACD_{self.fast_macd}_{self.slow_macd}_{self.signal_macd}"] + + long_condition = (bbp < self.bb_long_threshold) & (macdh > 0) & (macd < 0) + short_condition = (bbp > self.bb_short_threshold) & (macdh < 0) & (macd > 0) + + df["side"] = 0 + df.loc[long_condition, "side"] = 1 + df.loc[short_condition, "side"] = -1 + return df From 90389e7e9510daa761d45c0a88775e28330e1693 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 20:37:52 +0100 Subject: [PATCH 21/64] (feat) add strategy_optimizer with optuna --- quants_lab/strategy/strategy_optimizer.py | 72 +++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 quants_lab/strategy/strategy_optimizer.py diff --git a/quants_lab/strategy/strategy_optimizer.py b/quants_lab/strategy/strategy_optimizer.py new file mode 100644 index 0000000..bd3534b --- /dev/null +++ b/quants_lab/strategy/strategy_optimizer.py @@ -0,0 +1,72 @@ +import optuna +from quants_lab.backtesting.backtesting import Backtesting +from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis +from quants_lab.strategy.mean_reversion.bollinger import Bollinger +from quants_lab.strategy.mean_reversion.macd_bb import MACDBB +from optuna.exceptions import TrialPruned + +STUDY_NAME = "bollinger" + + +def objective(trial): + strategy = Bollinger( + exchange="binance_perpetual", + trading_pair="ETH-USDT", + interval="3m", + bb_length=trial.suggest_int("bb_length", 20, 300), + bb_std=trial.suggest_float("bb_std", 1.0, 3.0), + bb_long_threshold=trial.suggest_float("bb_long_threshold", -0.5, 0.3), + bb_short_threshold=trial.suggest_float("bb_short_threshold", 0.7, 1.5), + ) + + # fast_macd = trial.suggest_int("fast_macd", 10, 50) + # strategy = MACDBB( + # exchange="binance_perpetual", + # trading_pair="ETH-USDT", + # interval="3m", + # bb_length=trial.suggest_int("bb_length", 20, 300), + # bb_std=trial.suggest_float("bb_std", 1.0, 3.0), + # bb_long_threshold=trial.suggest_float("bb_long_threshold", -0.5, 0.3), + # bb_short_threshold=trial.suggest_float("bb_short_threshold", 0.7, 1.5), + # fast_macd=fast_macd, + # slow_macd=trial.suggest_int("slow_macd", fast_macd + 1, 100), + # signal_macd=trial.suggest_int("signal_macd", 8, 54) + # + # ) + try: + backtesting = Backtesting(strategy=strategy) + backtesting_result = backtesting.run_backtesting( + order_amount=50, + leverage=20, + initial_portfolio=100, + take_profit_multiplier=trial.suggest_float("take_profit_multiplier", 1.0, 5.0), + stop_loss_multiplier=trial.suggest_float("stop_loss_multiplier", 1.0, 5.0), + time_limit=60 * 60 * trial.suggest_int("time_limit", 1, 24), + std_span=None, + ) + backtesting_analysis = BacktestingAnalysis( + positions=backtesting_result, + ) + + trial.set_user_attr("net_profit_usd", backtesting_analysis.net_profit_usd()) + trial.set_user_attr("net_profit_pct", backtesting_analysis.net_profit_pct()) + trial.set_user_attr("max_drawdown_usd", backtesting_analysis.max_drawdown_usd()) + trial.set_user_attr("max_drawdown_pct", backtesting_analysis.max_drawdown_pct()) + trial.set_user_attr("sharpe_ratio", backtesting_analysis.sharpe_ratio()) + trial.set_user_attr("accuracy", backtesting_analysis.accuracy()) + trial.set_user_attr("total_positions", backtesting_analysis.total_positions()) + trial.set_user_attr("win_signals", backtesting_analysis.win_signals().shape[0]) + trial.set_user_attr("loss_signals", backtesting_analysis.loss_signals().shape[0]) + trial.set_user_attr("profit_factor", backtesting_analysis.profit_factor()) + trial.set_user_attr("duration_in_hours", backtesting_analysis.duration_in_minutes() / 60) + trial.set_user_attr("avg_trading_time_in_hours", backtesting_analysis.avg_trading_time_in_minutes() / 60) + return backtesting_analysis.net_profit_pct() + except Exception as e: + print(e) + raise TrialPruned() + + +study = optuna.create_study(direction="maximize", study_name=STUDY_NAME, storage="sqlite:///backtesting_report.db", + load_if_exists=True) + +study.optimize(objective, n_trials=2000) From 541b3e79e8ecac1d17b7c3f4218d2965999a160e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 20:38:11 +0100 Subject: [PATCH 22/64] (feat) refactor analysis page and notebook --- pages/9_⚙️_Backtesting.py | 36 +++---- .../mean_reversion_strat.ipynb | 95 ++++++++----------- 2 files changed, 57 insertions(+), 74 deletions(-) diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py index 0fd02f5..0e2730d 100644 --- a/pages/9_⚙️_Backtesting.py +++ b/pages/9_⚙️_Backtesting.py @@ -2,6 +2,7 @@ import pandas as pd import pandas_ta as ta import streamlit as st +from quants_lab.strategy.mean_reversion.bollinger import Bollinger from quants_lab.utils import data_management from quants_lab.backtesting.backtesting import Backtesting from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis @@ -14,44 +15,37 @@ st.set_page_config( ) st.title("⚙️ Backtesting") - -df = data_management.get_dataframe( - exchange='binance_perpetual', - trading_pair='BTC-USDT', - interval='1m', -) - df_to_show = data_management.get_dataframe( exchange='binance_perpetual', - trading_pair='BTC-USDT', + trading_pair='IOTA-USDT', interval='1h', ) -def bbands_strategy(df): - df.ta.bbands(length=100, std=3, append=True) - df["side"] = 0 - long_condition = df["BBP_100_3.0"] < -0.2 - short_condition = df["BBP_100_3.0"] > 1.2 - df.loc[long_condition, "side"] = 1 - df.loc[short_condition, "side"] = -1 - return df +strategy = Bollinger( + exchange="binance_perpetual", + trading_pair="ETH-USDT", + interval="1h", + bb_length=24, + bb_std=2.0, +) + +backtesting = Backtesting(strategy=strategy) -backtesting = Backtesting(candles_df=df) backtesting_result = backtesting.run_backtesting( - strategy=bbands_strategy, order_amount=50, leverage=20, initial_portfolio=100, - take_profit_multiplier=1.5, - stop_loss_multiplier=0.8, - time_limit=60 * 60 * 24, + take_profit_multiplier=3.5, + stop_loss_multiplier=1.5, + time_limit=60 * 60 * 12, std_span=None, ) backtesting_analysis = BacktestingAnalysis(df_to_show, backtesting_result, extra_rows=1, show_volume=False) backtesting_analysis.add_trade_pnl(row=2) +# backtesting_analysis.add_positions() c1, c2 = st.columns([0.2, 0.8]) with c1: diff --git a/quants_lab/research_notebooks/mean_reversion_strat.ipynb b/quants_lab/research_notebooks/mean_reversion_strat.ipynb index e96348f..d01e402 100644 --- a/quants_lab/research_notebooks/mean_reversion_strat.ipynb +++ b/quants_lab/research_notebooks/mean_reversion_strat.ipynb @@ -6,8 +6,8 @@ "metadata": { "collapsed": true, "ExecuteTime": { - "end_time": "2023-06-23T18:54:06.001204Z", - "start_time": "2023-06-23T18:54:05.418412Z" + "end_time": "2023-06-28T16:41:52.527419Z", + "start_time": "2023-06-28T16:41:51.952891Z" } }, "outputs": [], @@ -21,23 +21,43 @@ "df = data_management.get_dataframe(\n", " exchange='binance_perpetual',\n", " trading_pair='ETH-USDT',\n", - " interval='3m',\n", + " interval='1h',\n", ")\n", "\n", "def bbands_strategy(df):\n", - " df.ta.bbands(length=100, std=3, append=True)\n", + " df.ta.bbands(length=24, std=2, append=True)\n", " df[\"side\"] = 0\n", - " long_condition = df[\"BBP_100_3.0\"] < 0.0\n", - " short_condition = df[\"BBP_100_3.0\"] > 1.0\n", + " long_condition = df[\"BBP_24_2.0\"] < 0.05\n", + " short_condition = df[\"BBP_24_2.0\"] > 0.95\n", " df.loc[long_condition, \"side\"] = 1\n", " df.loc[short_condition, \"side\"] = -1\n", - " return df" + " return df\n" ] }, { "cell_type": "code", "execution_count": 2, - "outputs": [], + "outputs": [ + { + "ename": "KeyError", + "evalue": "\"None of [Index([(True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, ...), (False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, ...)], dtype='object')] are in the [index]\"", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mKeyError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[2], line 1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[43mmean_reversion_strategy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_data\u001B[49m\u001B[43m(\u001B[49m\u001B[43mstart\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43m2021-01-01\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43m2021-01-31\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/Documents/work/dashboard/quants_lab/strategy/directional_strategy_base.py:9\u001B[0m, in \u001B[0;36mDirectionalStrategyBase.get_data\u001B[0;34m(self, start, end)\u001B[0m\n\u001B[1;32m 7\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mget_data\u001B[39m(\u001B[38;5;28mself\u001B[39m, start: Optional[\u001B[38;5;28mstr\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m, end: Optional[\u001B[38;5;28mstr\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[1;32m 8\u001B[0m df \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mget_raw_data()\n\u001B[0;32m----> 9\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfilter_df_by_time\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstart\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/Documents/work/dashboard/quants_lab/strategy/directional_strategy_base.py:28\u001B[0m, in \u001B[0;36mDirectionalStrategyBase.filter_df_by_time\u001B[0;34m(df, start, end)\u001B[0m\n\u001B[1;32m 26\u001B[0m timeframe_conditions\u001B[38;5;241m.\u001B[39mappend(df[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimestamp\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m datetime\u001B[38;5;241m.\u001B[39mstrptime(end, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m%\u001B[39m\u001B[38;5;124mY-\u001B[39m\u001B[38;5;124m%\u001B[39m\u001B[38;5;124mm-\u001B[39m\u001B[38;5;132;01m%d\u001B[39;00m\u001B[38;5;124m\"\u001B[39m))\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(timeframe_conditions) \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[0;32m---> 28\u001B[0m df \u001B[38;5;241m=\u001B[39m \u001B[43mdf\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mloc\u001B[49m\u001B[43m[\u001B[49m\u001B[43mtimeframe_conditions\u001B[49m\u001B[43m]\u001B[49m\n\u001B[1;32m 29\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 30\u001B[0m df \u001B[38;5;241m=\u001B[39m df\u001B[38;5;241m.\u001B[39mcopy()\n", + "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1103\u001B[0m, in \u001B[0;36m_LocationIndexer.__getitem__\u001B[0;34m(self, key)\u001B[0m\n\u001B[1;32m 1100\u001B[0m axis \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39maxis \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 1102\u001B[0m maybe_callable \u001B[38;5;241m=\u001B[39m com\u001B[38;5;241m.\u001B[39mapply_if_callable(key, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj)\n\u001B[0;32m-> 1103\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_getitem_axis\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmaybe_callable\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1332\u001B[0m, in \u001B[0;36m_LocIndexer._getitem_axis\u001B[0;34m(self, key, axis)\u001B[0m\n\u001B[1;32m 1329\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mhasattr\u001B[39m(key, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mndim\u001B[39m\u001B[38;5;124m\"\u001B[39m) \u001B[38;5;129;01mand\u001B[39;00m key\u001B[38;5;241m.\u001B[39mndim \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m 1330\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mCannot index with multidimensional key\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m-> 1332\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_getitem_iterable\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1334\u001B[0m \u001B[38;5;66;03m# nested tuple slicing\u001B[39;00m\n\u001B[1;32m 1335\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m is_nested_tuple(key, labels):\n", + "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1272\u001B[0m, in \u001B[0;36m_LocIndexer._getitem_iterable\u001B[0;34m(self, key, axis)\u001B[0m\n\u001B[1;32m 1269\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_validate_key(key, axis)\n\u001B[1;32m 1271\u001B[0m \u001B[38;5;66;03m# A collection of keys\u001B[39;00m\n\u001B[0;32m-> 1272\u001B[0m keyarr, indexer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_listlike_indexer\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1273\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj\u001B[38;5;241m.\u001B[39m_reindex_with_indexers(\n\u001B[1;32m 1274\u001B[0m {axis: [keyarr, indexer]}, copy\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m, allow_dups\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m\n\u001B[1;32m 1275\u001B[0m )\n", + "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1462\u001B[0m, in \u001B[0;36m_LocIndexer._get_listlike_indexer\u001B[0;34m(self, key, axis)\u001B[0m\n\u001B[1;32m 1459\u001B[0m ax \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj\u001B[38;5;241m.\u001B[39m_get_axis(axis)\n\u001B[1;32m 1460\u001B[0m axis_name \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj\u001B[38;5;241m.\u001B[39m_get_axis_name(axis)\n\u001B[0;32m-> 1462\u001B[0m keyarr, indexer \u001B[38;5;241m=\u001B[39m \u001B[43max\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_indexer_strict\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis_name\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1464\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m keyarr, indexer\n", + "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexes/base.py:5876\u001B[0m, in \u001B[0;36mIndex._get_indexer_strict\u001B[0;34m(self, key, axis_name)\u001B[0m\n\u001B[1;32m 5873\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 5874\u001B[0m keyarr, indexer, new_indexer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_reindex_non_unique(keyarr)\n\u001B[0;32m-> 5876\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_raise_if_missing\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkeyarr\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mindexer\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis_name\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 5878\u001B[0m keyarr \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtake(indexer)\n\u001B[1;32m 5879\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(key, Index):\n\u001B[1;32m 5880\u001B[0m \u001B[38;5;66;03m# GH 42790 - Preserve name from an Index\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexes/base.py:5935\u001B[0m, in \u001B[0;36mIndex._raise_if_missing\u001B[0;34m(self, key, indexer, axis_name)\u001B[0m\n\u001B[1;32m 5933\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m use_interval_msg:\n\u001B[1;32m 5934\u001B[0m key \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(key)\n\u001B[0;32m-> 5935\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mKeyError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNone of [\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mkey\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m] are in the [\u001B[39m\u001B[38;5;132;01m{\u001B[39;00maxis_name\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m]\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 5937\u001B[0m not_found \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(ensure_index(key)[missing_mask\u001B[38;5;241m.\u001B[39mnonzero()[\u001B[38;5;241m0\u001B[39m]]\u001B[38;5;241m.\u001B[39munique())\n\u001B[1;32m 5938\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mKeyError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mnot_found\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m not in index\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", + "\u001B[0;31mKeyError\u001B[0m: \"None of [Index([(True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, ...), (False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, ...)], dtype='object')] are in the [index]\"" + ] + } + ], "source": [ "backtesting = Backtesting(candles_df=df)\n", "\n", @@ -46,9 +66,9 @@ " order_amount=50,\n", " leverage=20,\n", " initial_portfolio=100,\n", - " take_profit_multiplier=0.5,\n", - " stop_loss_multiplier=5.0,\n", - " time_limit=60 * 60 * 1,\n", + " take_profit_multiplier=3.0,\n", + " stop_loss_multiplier=1.2,\n", + " time_limit=60 * 60 * 24,\n", " std_span=None,\n", ")\n", "backtesting_report = BacktestingAnalysis(df, positions, extra_rows=1, show_volume=False)\n" @@ -56,33 +76,22 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-06-23T18:54:08.381976Z", - "start_time": "2023-06-23T18:54:06.002029Z" + "end_time": "2023-06-28T16:42:00.760306Z", + "start_time": "2023-06-28T16:41:59.886587Z" } } }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 14, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Strategy Performance Report:\n", - " - Net Profit: -4.07 USD (-4.07%)\n", - " - Total Positions: 978\n", - " - Win Signals: 581\n", - " - Loss Signals: 397\n", - " - Accuracy: 0.59%\n", - " - Profit Factor: 0.97\n", - " - Max Drawdown: -13.85 USD | -13.85%\n", - " - Sharpe Ratio: -0.01\n", - " - Duration: 6,478.55 Hours\n", - " - Average Trade Duration: 48.62 minutes\n", - " \n" - ] + "data": { + "text/plain": "timestamp datetime64[ns]\nopen float64\nhigh float64\nlow float64\nclose float64\nvolume float64\nquote_asset_volume float64\nn_trades float64\ntaker_buy_base_volume float64\ntaker_buy_quote_volume float64\ndtype: object" + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -91,28 +100,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-06-23T18:54:09.658773Z", - "start_time": "2023-06-23T18:54:09.628910Z" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "# Set the backend to Plotly\n", - "pd.options.plotting.backend = \"plotly\"\n", - "\n", - "positions[\"ret_usd\"].cumsum().plot()" - ], - "metadata": { - "collapsed": false, - "is_executing": true, - "ExecuteTime": { - "start_time": "2023-06-23T18:54:11.277722Z" + "end_time": "2023-06-28T16:29:06.142809Z", + "start_time": "2023-06-28T16:29:06.139861Z" } } }, From 9dcc9637b93787070f4f93c51bdd18ab6ec30aa6 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 22:21:19 +0100 Subject: [PATCH 23/64] (fix) filter timestamp condition --- quants_lab/strategy/directional_strategy_base.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index d878df8..c749208 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -19,13 +19,12 @@ class DirectionalStrategyBase: @staticmethod def filter_df_by_time(df, start: Optional[str] = None, end: Optional[str] = None): - timeframe_conditions = [] if start is not None: - timeframe_conditions.append(df["timestamp"] >= datetime.strptime(start, "%Y-%m-%d")) - if end is not None: - timeframe_conditions.append(df["timestamp"] <= datetime.strptime(end, "%Y-%m-%d")) - if len(timeframe_conditions) > 0: - df = df.loc[timeframe_conditions[0] & timeframe_conditions[1]] + start_condition = df["timestamp"] >= datetime.strptime(start, "%Y-%m-%d") else: - df = df.copy() - return df \ No newline at end of file + start_condition = True + if end is not None: + end_condition = df["timestamp"] <= datetime.strptime(end, "%Y-%m-%d") + else: + end_condition = True + return df.loc[start_condition & end_condition] From f8c778ac3ef7e6a1b9c9d3207332bb4966dcf2f4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 22:21:35 +0100 Subject: [PATCH 24/64] (fix) hardcoded threshold --- quants_lab/strategy/mean_reversion/bollinger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quants_lab/strategy/mean_reversion/bollinger.py b/quants_lab/strategy/mean_reversion/bollinger.py index 9df5aaa..6a7a7e4 100644 --- a/quants_lab/strategy/mean_reversion/bollinger.py +++ b/quants_lab/strategy/mean_reversion/bollinger.py @@ -36,7 +36,7 @@ class Bollinger(DirectionalStrategyBase): def add_signals(self, df): df["side"] = 0 long_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] < self.bb_long_threshold - short_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] > 0.95 + short_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] > self.bb_short_threshold df.loc[long_condition, "side"] = 1 df.loc[short_condition, "side"] = -1 return df From a47731af9e39574abc6f61def209b033f309c900 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 22:21:46 +0100 Subject: [PATCH 25/64] (fix) ohlc not showing --- quants_lab/backtesting/backtesting_analysis.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/quants_lab/backtesting/backtesting_analysis.py b/quants_lab/backtesting/backtesting_analysis.py index 35baa61..095e907 100644 --- a/quants_lab/backtesting/backtesting_analysis.py +++ b/quants_lab/backtesting/backtesting_analysis.py @@ -11,8 +11,9 @@ class BacktestingAnalysis: def __init__(self, positions: pd.DataFrame, candles_df: Optional[pd.DataFrame] = None): self.candles_df = candles_df self.positions = positions + self.base_figure = None - def create_base_figure(self, candlestick=True, volume=True, extra_rows=1): + def create_base_figure(self, candlestick=True, volume=True, positions=False, extra_rows=1): rows, heights = self.get_n_rows_and_heights(extra_rows, volume) self.rows = rows specs = [[{"secondary_y": True}]] * rows @@ -22,6 +23,8 @@ class BacktestingAnalysis: self.add_candles_graph() if volume: self.add_volume() + if positions: + self.add_positions() self.update_layout(volume) def add_positions(self): @@ -38,7 +41,8 @@ class BacktestingAnalysis: marker_color=active_signals.loc[(active_signals['side'] != 0), 'color'], marker_symbol=active_signals.loc[(active_signals['side'] != 0), 'symbol'], marker_size=20, - marker_line={'color': 'black', 'width': 0.7})) + marker_line={'color': 'black', 'width': 0.7}), + row=1, col=1) for index, row in active_signals.iterrows(): self.base_figure.add_shape(type="rect", @@ -48,7 +52,8 @@ class BacktestingAnalysis: y0=row.close, x1=row.close_time, y1=row.tp, - line=dict(color="green")) + line=dict(color="green"), + row=1, col=1) # Add SL self.base_figure.add_shape(type="rect", fillcolor="red", @@ -57,7 +62,8 @@ class BacktestingAnalysis: y0=row.close, x1=row.close_time, y1=row.sl, - line=dict(color="red")) + line=dict(color="red"), + row=1, col=1) def get_n_rows_and_heights(self, extra_rows, volume=True): rows = 1 + extra_rows + volume From cb2defc3f745977f84ade53b26bd4a4eee836c79 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 29 Jun 2023 22:21:59 +0100 Subject: [PATCH 26/64] (fix) use same trading pair --- pages/9_⚙️_Backtesting.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py index 0e2730d..20b6536 100644 --- a/pages/9_⚙️_Backtesting.py +++ b/pages/9_⚙️_Backtesting.py @@ -17,35 +17,37 @@ st.title("⚙️ Backtesting") df_to_show = data_management.get_dataframe( exchange='binance_perpetual', - trading_pair='IOTA-USDT', + trading_pair="ETH-USDT", interval='1h', ) + strategy = Bollinger( - exchange="binance_perpetual", - trading_pair="ETH-USDT", - interval="1h", - bb_length=24, - bb_std=2.0, -) + exchange="binance_perpetual", + trading_pair="ETH-USDT", + interval="3m", + bb_length=66, + bb_std=2.8, + bb_long_threshold=0.17, + bb_short_threshold=1.23, + ) backtesting = Backtesting(strategy=strategy) - -backtesting_result = backtesting.run_backtesting( +positions = backtesting.run_backtesting( + start='2021-04-01', + # end='2023-06-02', order_amount=50, leverage=20, initial_portfolio=100, - take_profit_multiplier=3.5, - stop_loss_multiplier=1.5, - time_limit=60 * 60 * 12, + take_profit_multiplier=4.3, + stop_loss_multiplier=3.0, + time_limit=60 * 60 * 24, std_span=None, ) - - -backtesting_analysis = BacktestingAnalysis(df_to_show, backtesting_result, extra_rows=1, show_volume=False) +backtesting_analysis = BacktestingAnalysis(positions=positions, candles_df=df_to_show) +backtesting_analysis.create_base_figure(volume=False, positions=True, extra_rows=1) backtesting_analysis.add_trade_pnl(row=2) -# backtesting_analysis.add_positions() c1, c2 = st.columns([0.2, 0.8]) with c1: From f7d4a0b5f0c0b82c8781f586481f400b2f9ca085 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 12 Jul 2023 15:31:21 +0200 Subject: [PATCH 27/64] (feat) update dependencies --- environment_conda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment_conda.yml b/environment_conda.yml index 1ee512f..4d9cdcb 100644 --- a/environment_conda.yml +++ b/environment_conda.yml @@ -6,7 +6,6 @@ dependencies: - sqlalchemy - pip - pip: - - ccxt - streamlit - watchdog - plotly @@ -18,5 +17,7 @@ dependencies: - pyyaml - commlib-py - jupyter + - optuna + - optuna-dashboard - git+https://github.com/hummingbot/hbot-remote-client-py.git - git+https://github.com/hummingbot/docker-manager.git From 99a0641ebd1c716fa74ddc78a96aff570afa5ac9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 12 Jul 2023 15:31:29 +0200 Subject: [PATCH 28/64] (feat) remove notebooks --- quants_lab/research_notebooks/__init__.py | 0 .../mean_reversion_strat.ipynb | 139 ------------------ 2 files changed, 139 deletions(-) delete mode 100644 quants_lab/research_notebooks/__init__.py delete mode 100644 quants_lab/research_notebooks/mean_reversion_strat.ipynb diff --git a/quants_lab/research_notebooks/__init__.py b/quants_lab/research_notebooks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/quants_lab/research_notebooks/mean_reversion_strat.ipynb b/quants_lab/research_notebooks/mean_reversion_strat.ipynb deleted file mode 100644 index d01e402..0000000 --- a/quants_lab/research_notebooks/mean_reversion_strat.ipynb +++ /dev/null @@ -1,139 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true, - "ExecuteTime": { - "end_time": "2023-06-28T16:41:52.527419Z", - "start_time": "2023-06-28T16:41:51.952891Z" - } - }, - "outputs": [], - "source": [ - "import pandas_ta as ta\n", - "\n", - "from quants_lab.utils import data_management\n", - "from quants_lab.backtesting.backtesting import Backtesting\n", - "from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis\n", - "\n", - "df = data_management.get_dataframe(\n", - " exchange='binance_perpetual',\n", - " trading_pair='ETH-USDT',\n", - " interval='1h',\n", - ")\n", - "\n", - "def bbands_strategy(df):\n", - " df.ta.bbands(length=24, std=2, append=True)\n", - " df[\"side\"] = 0\n", - " long_condition = df[\"BBP_24_2.0\"] < 0.05\n", - " short_condition = df[\"BBP_24_2.0\"] > 0.95\n", - " df.loc[long_condition, \"side\"] = 1\n", - " df.loc[short_condition, \"side\"] = -1\n", - " return df\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "outputs": [ - { - "ename": "KeyError", - "evalue": "\"None of [Index([(True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, ...), (False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, ...)], dtype='object')] are in the [index]\"", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mKeyError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[2], line 1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[43mmean_reversion_strategy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_data\u001B[49m\u001B[43m(\u001B[49m\u001B[43mstart\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43m2021-01-01\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43m2021-01-31\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/Documents/work/dashboard/quants_lab/strategy/directional_strategy_base.py:9\u001B[0m, in \u001B[0;36mDirectionalStrategyBase.get_data\u001B[0;34m(self, start, end)\u001B[0m\n\u001B[1;32m 7\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mget_data\u001B[39m(\u001B[38;5;28mself\u001B[39m, start: Optional[\u001B[38;5;28mstr\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m, end: Optional[\u001B[38;5;28mstr\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[1;32m 8\u001B[0m df \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mget_raw_data()\n\u001B[0;32m----> 9\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfilter_df_by_time\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstart\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mend\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/Documents/work/dashboard/quants_lab/strategy/directional_strategy_base.py:28\u001B[0m, in \u001B[0;36mDirectionalStrategyBase.filter_df_by_time\u001B[0;34m(df, start, end)\u001B[0m\n\u001B[1;32m 26\u001B[0m timeframe_conditions\u001B[38;5;241m.\u001B[39mappend(df[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimestamp\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m datetime\u001B[38;5;241m.\u001B[39mstrptime(end, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m%\u001B[39m\u001B[38;5;124mY-\u001B[39m\u001B[38;5;124m%\u001B[39m\u001B[38;5;124mm-\u001B[39m\u001B[38;5;132;01m%d\u001B[39;00m\u001B[38;5;124m\"\u001B[39m))\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(timeframe_conditions) \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[0;32m---> 28\u001B[0m df \u001B[38;5;241m=\u001B[39m \u001B[43mdf\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mloc\u001B[49m\u001B[43m[\u001B[49m\u001B[43mtimeframe_conditions\u001B[49m\u001B[43m]\u001B[49m\n\u001B[1;32m 29\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 30\u001B[0m df \u001B[38;5;241m=\u001B[39m df\u001B[38;5;241m.\u001B[39mcopy()\n", - "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1103\u001B[0m, in \u001B[0;36m_LocationIndexer.__getitem__\u001B[0;34m(self, key)\u001B[0m\n\u001B[1;32m 1100\u001B[0m axis \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39maxis \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 1102\u001B[0m maybe_callable \u001B[38;5;241m=\u001B[39m com\u001B[38;5;241m.\u001B[39mapply_if_callable(key, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj)\n\u001B[0;32m-> 1103\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_getitem_axis\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmaybe_callable\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1332\u001B[0m, in \u001B[0;36m_LocIndexer._getitem_axis\u001B[0;34m(self, key, axis)\u001B[0m\n\u001B[1;32m 1329\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mhasattr\u001B[39m(key, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mndim\u001B[39m\u001B[38;5;124m\"\u001B[39m) \u001B[38;5;129;01mand\u001B[39;00m key\u001B[38;5;241m.\u001B[39mndim \u001B[38;5;241m>\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m 1330\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mCannot index with multidimensional key\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m-> 1332\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_getitem_iterable\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1334\u001B[0m \u001B[38;5;66;03m# nested tuple slicing\u001B[39;00m\n\u001B[1;32m 1335\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m is_nested_tuple(key, labels):\n", - "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1272\u001B[0m, in \u001B[0;36m_LocIndexer._getitem_iterable\u001B[0;34m(self, key, axis)\u001B[0m\n\u001B[1;32m 1269\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_validate_key(key, axis)\n\u001B[1;32m 1271\u001B[0m \u001B[38;5;66;03m# A collection of keys\u001B[39;00m\n\u001B[0;32m-> 1272\u001B[0m keyarr, indexer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_listlike_indexer\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1273\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj\u001B[38;5;241m.\u001B[39m_reindex_with_indexers(\n\u001B[1;32m 1274\u001B[0m {axis: [keyarr, indexer]}, copy\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m, allow_dups\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m\n\u001B[1;32m 1275\u001B[0m )\n", - "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexing.py:1462\u001B[0m, in \u001B[0;36m_LocIndexer._get_listlike_indexer\u001B[0;34m(self, key, axis)\u001B[0m\n\u001B[1;32m 1459\u001B[0m ax \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj\u001B[38;5;241m.\u001B[39m_get_axis(axis)\n\u001B[1;32m 1460\u001B[0m axis_name \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mobj\u001B[38;5;241m.\u001B[39m_get_axis_name(axis)\n\u001B[0;32m-> 1462\u001B[0m keyarr, indexer \u001B[38;5;241m=\u001B[39m \u001B[43max\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_indexer_strict\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis_name\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1464\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m keyarr, indexer\n", - "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexes/base.py:5876\u001B[0m, in \u001B[0;36mIndex._get_indexer_strict\u001B[0;34m(self, key, axis_name)\u001B[0m\n\u001B[1;32m 5873\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 5874\u001B[0m keyarr, indexer, new_indexer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_reindex_non_unique(keyarr)\n\u001B[0;32m-> 5876\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_raise_if_missing\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkeyarr\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mindexer\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43maxis_name\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 5878\u001B[0m keyarr \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtake(indexer)\n\u001B[1;32m 5879\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(key, Index):\n\u001B[1;32m 5880\u001B[0m \u001B[38;5;66;03m# GH 42790 - Preserve name from an Index\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/dashboard/lib/python3.9/site-packages/pandas/core/indexes/base.py:5935\u001B[0m, in \u001B[0;36mIndex._raise_if_missing\u001B[0;34m(self, key, indexer, axis_name)\u001B[0m\n\u001B[1;32m 5933\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m use_interval_msg:\n\u001B[1;32m 5934\u001B[0m key \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(key)\n\u001B[0;32m-> 5935\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mKeyError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNone of [\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mkey\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m] are in the [\u001B[39m\u001B[38;5;132;01m{\u001B[39;00maxis_name\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m]\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 5937\u001B[0m not_found \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlist\u001B[39m(ensure_index(key)[missing_mask\u001B[38;5;241m.\u001B[39mnonzero()[\u001B[38;5;241m0\u001B[39m]]\u001B[38;5;241m.\u001B[39munique())\n\u001B[1;32m 5938\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mKeyError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mnot_found\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m not in index\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", - "\u001B[0;31mKeyError\u001B[0m: \"None of [Index([(True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, ...), (False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, ...)], dtype='object')] are in the [index]\"" - ] - } - ], - "source": [ - "backtesting = Backtesting(candles_df=df)\n", - "\n", - "positions = backtesting.run_backtesting(\n", - " strategy=bbands_strategy,\n", - " order_amount=50,\n", - " leverage=20,\n", - " initial_portfolio=100,\n", - " take_profit_multiplier=3.0,\n", - " stop_loss_multiplier=1.2,\n", - " time_limit=60 * 60 * 24,\n", - " std_span=None,\n", - ")\n", - "backtesting_report = BacktestingAnalysis(df, positions, extra_rows=1, show_volume=False)\n" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-06-28T16:42:00.760306Z", - "start_time": "2023-06-28T16:41:59.886587Z" - } - } - }, - { - "cell_type": "code", - "execution_count": 14, - "outputs": [ - { - "data": { - "text/plain": "timestamp datetime64[ns]\nopen float64\nhigh float64\nlow float64\nclose float64\nvolume float64\nquote_asset_volume float64\nn_trades float64\ntaker_buy_base_volume float64\ntaker_buy_quote_volume float64\ndtype: object" - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "print(backtesting_report.text_report())" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-06-28T16:29:06.142809Z", - "start_time": "2023-06-28T16:29:06.139861Z" - } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], - "metadata": { - "collapsed": false - } - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} From 00bbf311acd7ca925fa7782a70698880b28ac2eb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 12 Jul 2023 15:31:45 +0200 Subject: [PATCH 29/64] (feat) add statistical arbitrage strategy --- .../strategy/mean_reversion/stat_arb.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 quants_lab/strategy/mean_reversion/stat_arb.py diff --git a/quants_lab/strategy/mean_reversion/stat_arb.py b/quants_lab/strategy/mean_reversion/stat_arb.py new file mode 100644 index 0000000..3c6f629 --- /dev/null +++ b/quants_lab/strategy/mean_reversion/stat_arb.py @@ -0,0 +1,51 @@ +import pandas as pd +import pandas_ta as ta +from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase + +from quants_lab.utils import data_management + + +class StatArb(DirectionalStrategyBase): + def __init__(self, + exchange="binance_perpetual", + trading_pair="DOGE-USDT", + target_trading_pair="BTC-USDT", + interval="1h", + periods=100, + deviation_threshold=1.1): + self.exchange = exchange + self.trading_pair = trading_pair + self.interval = interval + self.target_trading_pair = target_trading_pair + self.periods = periods + self.deviation_threshold = deviation_threshold + + def get_raw_data(self): + df = data_management.get_dataframe( + exchange=self.exchange, + trading_pair=self.trading_pair, + interval=self.interval, + ) + df_target = data_management.get_dataframe( + exchange=self.exchange, + trading_pair=self.target_trading_pair, + interval=self.interval, + ) + df = pd.merge(df, df_target, on="timestamp", how='inner', suffixes=('', '_target')) + return df + + def add_indicators(self, df): + df["pct_change_original"] = df["close"].pct_change() + df["pct_change_target"] = df["close_target"].pct_change() + df["spread"] = df["pct_change_target"] - df["pct_change_original"] + df["cum_spread"] = df["spread"].rolling(self.periods).sum() + df["z_score"] = ta.zscore(df["cum_spread"], length=self.periods) + return df + + def add_signals(self, df): + df["side"] = 0 + short_condition = df["z_score"] < - self.deviation_threshold + long_condition = df["z_score"] > self.deviation_threshold + df.loc[long_condition, "side"] = 1 + df.loc[short_condition, "side"] = -1 + return df From 298988a68f6aa55f447d6a0123f69e1a0c1501ba Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 12 Jul 2023 15:36:45 +0200 Subject: [PATCH 30/64] (fix) improve time filter --- quants_lab/strategy/directional_strategy_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index c749208..f26ab37 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import Optional +import pandas as pd class DirectionalStrategyBase: @@ -22,9 +23,9 @@ class DirectionalStrategyBase: if start is not None: start_condition = df["timestamp"] >= datetime.strptime(start, "%Y-%m-%d") else: - start_condition = True + start_condition = pd.Series([True]*len(df)) if end is not None: end_condition = df["timestamp"] <= datetime.strptime(end, "%Y-%m-%d") else: - end_condition = True - return df.loc[start_condition & end_condition] + end_condition = pd.Series([True]*len(df)) + return df[start_condition & end_condition] From ff50b19ee24f43a8d9756288f6c0d4c5b59160ea Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 12 Jul 2023 15:36:59 +0200 Subject: [PATCH 31/64] (feat) test stat arb --- pages/9_⚙️_Backtesting.py | 17 ++++--------- quants_lab/strategy/strategy_optimizer.py | 31 +++++++++++++---------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py index 20b6536..39b1136 100644 --- a/pages/9_⚙️_Backtesting.py +++ b/pages/9_⚙️_Backtesting.py @@ -3,6 +3,7 @@ import pandas_ta as ta import streamlit as st from quants_lab.strategy.mean_reversion.bollinger import Bollinger +from quants_lab.strategy.mean_reversion.stat_arb import StatArb from quants_lab.utils import data_management from quants_lab.backtesting.backtesting import Backtesting from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis @@ -22,26 +23,18 @@ df_to_show = data_management.get_dataframe( ) -strategy = Bollinger( - exchange="binance_perpetual", - trading_pair="ETH-USDT", - interval="3m", - bb_length=66, - bb_std=2.8, - bb_long_threshold=0.17, - bb_short_threshold=1.23, - ) +strategy = StatArb(trading_pair="ETH-USDT", periods=24, deviation_threshold=1.5) backtesting = Backtesting(strategy=strategy) positions = backtesting.run_backtesting( - start='2021-04-01', + # start='2022-01-01', # end='2023-06-02', order_amount=50, leverage=20, initial_portfolio=100, - take_profit_multiplier=4.3, - stop_loss_multiplier=3.0, + take_profit_multiplier=3.0, + stop_loss_multiplier=1.5, time_limit=60 * 60 * 24, std_span=None, ) diff --git a/quants_lab/strategy/strategy_optimizer.py b/quants_lab/strategy/strategy_optimizer.py index bd3534b..bfbf441 100644 --- a/quants_lab/strategy/strategy_optimizer.py +++ b/quants_lab/strategy/strategy_optimizer.py @@ -5,19 +5,21 @@ from quants_lab.strategy.mean_reversion.bollinger import Bollinger from quants_lab.strategy.mean_reversion.macd_bb import MACDBB from optuna.exceptions import TrialPruned -STUDY_NAME = "bollinger" +from quants_lab.strategy.mean_reversion.stat_arb import StatArb + +STUDY_NAME = "stat_arb" def objective(trial): - strategy = Bollinger( - exchange="binance_perpetual", - trading_pair="ETH-USDT", - interval="3m", - bb_length=trial.suggest_int("bb_length", 20, 300), - bb_std=trial.suggest_float("bb_std", 1.0, 3.0), - bb_long_threshold=trial.suggest_float("bb_long_threshold", -0.5, 0.3), - bb_short_threshold=trial.suggest_float("bb_short_threshold", 0.7, 1.5), - ) + # strategy = Bollinger( + # exchange="binance_perpetual", + # trading_pair="ETH-USDT", + # interval="3m", + # bb_length=trial.suggest_int("bb_length", 20, 300), + # bb_std=trial.suggest_float("bb_std", 1.0, 3.0), + # bb_long_threshold=trial.suggest_float("bb_long_threshold", -0.5, 0.3), + # bb_short_threshold=trial.suggest_float("bb_short_threshold", 0.7, 1.5), + # ) # fast_macd = trial.suggest_int("fast_macd", 10, 50) # strategy = MACDBB( @@ -31,16 +33,19 @@ def objective(trial): # fast_macd=fast_macd, # slow_macd=trial.suggest_int("slow_macd", fast_macd + 1, 100), # signal_macd=trial.suggest_int("signal_macd", 8, 54) - # # ) + strategy = StatArb(trading_pair="ETH-USDT", + periods=trial.suggest_int("periods", 10, 150), + deviation_threshold=trial.suggest_float("deviation_threshold", 0.9, 2.0)) try: backtesting = Backtesting(strategy=strategy) backtesting_result = backtesting.run_backtesting( + start='2021-04-01', order_amount=50, leverage=20, initial_portfolio=100, - take_profit_multiplier=trial.suggest_float("take_profit_multiplier", 1.0, 5.0), - stop_loss_multiplier=trial.suggest_float("stop_loss_multiplier", 1.0, 5.0), + take_profit_multiplier=trial.suggest_float("take_profit_multiplier", 1.0, 3.0), + stop_loss_multiplier=trial.suggest_float("stop_loss_multiplier", 1.0, 3.0), time_limit=60 * 60 * trial.suggest_int("time_limit", 1, 24), std_span=None, ) From ec49fbd33c9b1a7a78037b712cbf4791148ad2eb Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Jul 2023 22:12:40 +0200 Subject: [PATCH 32/64] (feat) remove utils --- quants_lab/utils/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 quants_lab/utils/__init__.py diff --git a/quants_lab/utils/__init__.py b/quants_lab/utils/__init__.py deleted file mode 100644 index e69de29..0000000 From 5c23a2dccaa2e0c628483c9a4514f22f677ceb74 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Jul 2023 22:12:48 +0200 Subject: [PATCH 33/64] (feat) remove data management --- quants_lab/utils/data_management.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 quants_lab/utils/data_management.py diff --git a/quants_lab/utils/data_management.py b/quants_lab/utils/data_management.py deleted file mode 100644 index 4160b9e..0000000 --- a/quants_lab/utils/data_management.py +++ /dev/null @@ -1,22 +0,0 @@ -import os - -import pandas as pd - - -def get_dataframe(exchange: str, trading_pair: str, interval: str) -> pd.DataFrame: - """ - Get a dataframe of market data from the database. - :param exchange: Exchange name - :param trading_pair: Trading pair - :param interval: Interval of the data - :return: Dataframe of market data - """ - script_dir = os.path.dirname(os.path.abspath(__file__)) - data_dir = os.path.join(script_dir, "../../data/candles") - filename = f"candles_{exchange}_{trading_pair.upper()}_{interval}.csv" - file_path = os.path.join(data_dir, filename) - if not os.path.exists(file_path): - raise FileNotFoundError(f"File '{file_path}' does not exist.") - df = pd.read_csv(file_path) - df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms") - return df From 04c5f43ebb817d72e37cfa05a4908a9ea3435be4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Jul 2023 22:12:54 +0200 Subject: [PATCH 34/64] (feat) remove data backtesting --- quants_lab/backtesting/backtesting.py | 63 --------------------------- 1 file changed, 63 deletions(-) delete mode 100644 quants_lab/backtesting/backtesting.py diff --git a/quants_lab/backtesting/backtesting.py b/quants_lab/backtesting/backtesting.py deleted file mode 100644 index cedc75f..0000000 --- a/quants_lab/backtesting/backtesting.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import Optional - -import pandas as pd - -from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis -from quants_lab.labeling.triple_barrier_method import triple_barrier_method -from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase - - -class Backtesting: - def __init__(self, strategy: DirectionalStrategyBase): - self.strategy = strategy - - def run_backtesting(self, - order_amount, leverage, initial_portfolio, - take_profit_multiplier, stop_loss_multiplier, time_limit, - std_span, taker_fee=0.0003, maker_fee=0.00012, - start: Optional[str] = None, end: Optional[str] = None): - df = self.strategy.get_data(start=start, end=end) - df = self.strategy.add_indicators(df) - df = self.strategy.add_signals(df) - df = triple_barrier_method( - df=df, - std_span=std_span, - tp=take_profit_multiplier, - sl=stop_loss_multiplier, - tl=time_limit, - trade_cost=taker_fee * 2, - max_executors=1, - ) - - backtesting_analysis = self.get_backtest_analysis( - df=df, - maker_fee=maker_fee, - taker_fee=taker_fee, - order_amount=order_amount, - leverage=leverage, - initial_portfolio=initial_portfolio, - ) - return backtesting_analysis - - @staticmethod - def get_backtest_analysis( - df, - order_amount=15, - initial_portfolio=700, - leverage=20, - maker_fee=0.00012, - taker_fee=0.0003, - ): - # Set starting params - first_row = df.iloc[0].tolist() - first_row.extend([0, 0, 0, 0, 0, initial_portfolio]) - active_signals = df[df["active_signal"] == 1].copy() - active_signals.loc[:, "amount"] = order_amount - active_signals.loc[:, "margin_used"] = order_amount / leverage - active_signals.loc[:, "fee_pct"] = active_signals["close_type"].apply(lambda x: maker_fee + taker_fee if x == "tp" else taker_fee * 2) - active_signals.loc[:, "fee_usd"] = active_signals["fee_pct"] * active_signals["amount"] - active_signals.loc[:, "ret_usd"] = active_signals.apply(lambda x: (x["ret"] - x["fee_pct"]) * x["amount"], axis=1) - active_signals.loc[:, "current_portfolio"] = initial_portfolio + active_signals["ret_usd"].cumsum() - active_signals.loc[:, "current_portfolio"].fillna(method='ffill', inplace=True) - positions = pd.concat([pd.DataFrame([first_row], columns=active_signals.columns), active_signals]) - return positions.reset_index(drop=True) From 0d6888b310f29f545145d73ef8b9ef22e5ba9f39 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Jul 2023 22:13:12 +0200 Subject: [PATCH 35/64] (feat) add backtesting and data management to base class --- .../strategy/directional_strategy_base.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index f26ab37..ca6d86a 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -1,7 +1,10 @@ +import os from datetime import datetime from typing import Optional import pandas as pd +from quants_lab.labeling.triple_barrier_method import triple_barrier_method + class DirectionalStrategyBase: @@ -18,6 +21,25 @@ class DirectionalStrategyBase: def add_signals(self, df): raise NotImplemented + @staticmethod + def get_candles(exchange: str, trading_pair: str, interval: str) -> pd.DataFrame: + """ + Get a dataframe of market data from the database. + :param exchange: Exchange name + :param trading_pair: Trading pair + :param interval: Interval of the data + :return: Dataframe of market data + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + data_dir = os.path.join(script_dir, "../../data/candles") + filename = f"candles_{exchange}_{trading_pair.upper()}_{interval}.csv" + file_path = os.path.join(data_dir, filename) + if not os.path.exists(file_path): + raise FileNotFoundError(f"File '{file_path}' does not exist.") + df = pd.read_csv(file_path) + df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms") + return df + @staticmethod def filter_df_by_time(df, start: Optional[str] = None, end: Optional[str] = None): if start is not None: @@ -29,3 +51,36 @@ class DirectionalStrategyBase: else: end_condition = pd.Series([True]*len(df)) return df[start_condition & end_condition] + + def run_backtesting(self, + take_profit_multiplier, stop_loss_multiplier, time_limit, + std_span, order_amount=100, leverage=20, initial_portfolio=1000, + taker_fee=0.0003, maker_fee=0.00012, + start: Optional[str] = None, end: Optional[str] = None): + df = self.get_data(start=start, end=end) + df = self.add_indicators(df) + df = self.add_signals(df) + df = triple_barrier_method( + df=df, + std_span=std_span, + tp=take_profit_multiplier, + sl=stop_loss_multiplier, + tl=time_limit, + trade_cost=taker_fee * 2, + max_executors=1, + ) + + first_row = df.iloc[0].tolist() + first_row.extend([0, 0, 0, 0, 0, initial_portfolio]) + active_signals = df[df["active_signal"] == 1].copy() + active_signals.loc[:, "amount"] = order_amount + active_signals.loc[:, "margin_used"] = order_amount / leverage + active_signals.loc[:, "fee_pct"] = active_signals["close_type"].apply( + lambda x: maker_fee + taker_fee if x == "tp" else taker_fee * 2) + active_signals.loc[:, "fee_usd"] = active_signals["fee_pct"] * active_signals["amount"] + active_signals.loc[:, "ret_usd"] = active_signals.apply(lambda x: (x["ret"] - x["fee_pct"]) * x["amount"], + axis=1) + active_signals.loc[:, "current_portfolio"] = initial_portfolio + active_signals["ret_usd"].cumsum() + active_signals.loc[:, "current_portfolio"].fillna(method='ffill', inplace=True) + positions = pd.concat([pd.DataFrame([first_row], columns=active_signals.columns), active_signals]) + return df, positions.reset_index(drop=True) From 74c5c4b3818d12d498c0b489b189d43d7d9de1a5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Jul 2023 22:17:11 +0200 Subject: [PATCH 36/64] (feat) rename folder --- quants_lab/strategy/{mean_reversion => experiments}/__init__.py | 0 quants_lab/strategy/{mean_reversion => experiments}/bollinger.py | 0 quants_lab/strategy/{mean_reversion => experiments}/macd_bb.py | 0 quants_lab/strategy/{mean_reversion => experiments}/stat_arb.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename quants_lab/strategy/{mean_reversion => experiments}/__init__.py (100%) rename quants_lab/strategy/{mean_reversion => experiments}/bollinger.py (100%) rename quants_lab/strategy/{mean_reversion => experiments}/macd_bb.py (100%) rename quants_lab/strategy/{mean_reversion => experiments}/stat_arb.py (100%) diff --git a/quants_lab/strategy/mean_reversion/__init__.py b/quants_lab/strategy/experiments/__init__.py similarity index 100% rename from quants_lab/strategy/mean_reversion/__init__.py rename to quants_lab/strategy/experiments/__init__.py diff --git a/quants_lab/strategy/mean_reversion/bollinger.py b/quants_lab/strategy/experiments/bollinger.py similarity index 100% rename from quants_lab/strategy/mean_reversion/bollinger.py rename to quants_lab/strategy/experiments/bollinger.py diff --git a/quants_lab/strategy/mean_reversion/macd_bb.py b/quants_lab/strategy/experiments/macd_bb.py similarity index 100% rename from quants_lab/strategy/mean_reversion/macd_bb.py rename to quants_lab/strategy/experiments/macd_bb.py diff --git a/quants_lab/strategy/mean_reversion/stat_arb.py b/quants_lab/strategy/experiments/stat_arb.py similarity index 100% rename from quants_lab/strategy/mean_reversion/stat_arb.py rename to quants_lab/strategy/experiments/stat_arb.py From ac6064c4166afc27c410c2e6f2e3e6a91c5d9fc9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Jul 2023 22:17:39 +0200 Subject: [PATCH 37/64] (feat) remove backtesting folder --- quants_lab/backtesting/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 quants_lab/backtesting/__init__.py diff --git a/quants_lab/backtesting/__init__.py b/quants_lab/backtesting/__init__.py deleted file mode 100644 index e69de29..0000000 From df5c5dd3a6335c16653254004702d5c6f86b47a4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Jul 2023 22:17:48 +0200 Subject: [PATCH 38/64] (feat) move strategy analysis to strategy module --- .../backtesting_analysis.py => strategy/strategy_analysis.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename quants_lab/{backtesting/backtesting_analysis.py => strategy/strategy_analysis.py} (100%) diff --git a/quants_lab/backtesting/backtesting_analysis.py b/quants_lab/strategy/strategy_analysis.py similarity index 100% rename from quants_lab/backtesting/backtesting_analysis.py rename to quants_lab/strategy/strategy_analysis.py From c3e2b52fc9a61496fdb6abd38c83756ca569c4bd Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 14:53:41 +0200 Subject: [PATCH 39/64] (feat) rename methods --- quants_lab/strategy/directional_strategy_base.py | 9 +++++---- quants_lab/strategy/experiments/bollinger.py | 4 ++-- quants_lab/strategy/experiments/macd_bb.py | 4 ++-- quants_lab/strategy/experiments/stat_arb.py | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index ca6d86a..b75d049 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -15,10 +15,10 @@ class DirectionalStrategyBase: def get_raw_data(self): raise NotImplemented - def add_indicators(self, df): + def preprocessing(self, df): raise NotImplemented - def add_signals(self, df): + def predict(self, df): raise NotImplemented @staticmethod @@ -57,9 +57,10 @@ class DirectionalStrategyBase: std_span, order_amount=100, leverage=20, initial_portfolio=1000, taker_fee=0.0003, maker_fee=0.00012, start: Optional[str] = None, end: Optional[str] = None): + # TODO: Evaluate to move the get data outside the backtesting to optimize the performance. df = self.get_data(start=start, end=end) - df = self.add_indicators(df) - df = self.add_signals(df) + df = self.preprocessing(df) + df = self.predict(df) df = triple_barrier_method( df=df, std_span=std_span, diff --git a/quants_lab/strategy/experiments/bollinger.py b/quants_lab/strategy/experiments/bollinger.py index 6a7a7e4..af252b8 100644 --- a/quants_lab/strategy/experiments/bollinger.py +++ b/quants_lab/strategy/experiments/bollinger.py @@ -29,11 +29,11 @@ class Bollinger(DirectionalStrategyBase): ) return df - def add_indicators(self, df): + def preprocessing(self, df): df.ta.bbands(length=self.bb_length, std=self.bb_std, append=True) return df - def add_signals(self, df): + def predict(self, df): df["side"] = 0 long_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] < self.bb_long_threshold short_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] > self.bb_short_threshold diff --git a/quants_lab/strategy/experiments/macd_bb.py b/quants_lab/strategy/experiments/macd_bb.py index fb691b9..f6712d6 100644 --- a/quants_lab/strategy/experiments/macd_bb.py +++ b/quants_lab/strategy/experiments/macd_bb.py @@ -35,12 +35,12 @@ class MACDBB(DirectionalStrategyBase): ) return df - def add_indicators(self, df): + def preprocessing(self, df): df.ta.bbands(length=self.bb_length, std=self.bb_std, append=True) df.ta.macd(fast=self.fast_macd, slow=self.slow_macd, signal=self.signal_macd, append=True) return df - def add_signals(self, df): + def predict(self, df): bbp = df[f"BBP_{self.bb_length}_{self.bb_std}"] macdh = df[f"MACDh_{self.fast_macd}_{self.slow_macd}_{self.signal_macd}"] macd = df[f"MACD_{self.fast_macd}_{self.slow_macd}_{self.signal_macd}"] diff --git a/quants_lab/strategy/experiments/stat_arb.py b/quants_lab/strategy/experiments/stat_arb.py index 3c6f629..8ec47fd 100644 --- a/quants_lab/strategy/experiments/stat_arb.py +++ b/quants_lab/strategy/experiments/stat_arb.py @@ -34,7 +34,7 @@ class StatArb(DirectionalStrategyBase): df = pd.merge(df, df_target, on="timestamp", how='inner', suffixes=('', '_target')) return df - def add_indicators(self, df): + def preprocessing(self, df): df["pct_change_original"] = df["close"].pct_change() df["pct_change_target"] = df["close_target"].pct_change() df["spread"] = df["pct_change_target"] - df["pct_change_original"] @@ -42,7 +42,7 @@ class StatArb(DirectionalStrategyBase): df["z_score"] = ta.zscore(df["cum_spread"], length=self.periods) return df - def add_signals(self, df): + def predict(self, df): df["side"] = 0 short_condition = df["z_score"] < - self.deviation_threshold long_condition = df["z_score"] > self.deviation_threshold From 3e9876f0c38e3df0da5eea01fb342ebf2b9d0adc Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 14:54:00 +0200 Subject: [PATCH 40/64] (feat) add more os utils --- utils/os_utils.py | 54 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/utils/os_utils.py b/utils/os_utils.py index caaad2b..863d75f 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -1,6 +1,9 @@ -import os +import glob import subprocess - +import importlib.util +import inspect +import os +from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase # update this to the actual import import yaml @@ -26,3 +29,50 @@ def read_yaml_file(file_path): def directory_exists(directory: str): return os.path.exists(directory) + + +def save_file(name: str, content: str, path: str): + complete_file_path = os.path.join(path, name) + os.makedirs(path, exist_ok=True) + with open(complete_file_path, "w") as file: + file.write(content) + + +def load_file(path: str) -> str: + try: + with open(path, 'r') as file: + contents = file.read() + return contents + except FileNotFoundError: + print(f"File '{path}' not found.") + return "" + except IOError: + print(f"Error reading file '{path}'.") + return "" + + +def get_python_files_from_directory(directory: str) -> list: + py_files = glob.glob(directory + "/**/*.py", recursive=True) + py_files = [path for path in py_files if not path.endswith("__init__.py")] + return py_files + + +def load_strategies(path): + strategy_classes = [] + for filename in os.listdir(path): + if filename.endswith('.py'): + module_name = filename[:-3] # strip the .py to get the module name + file_path = os.path.join(path, filename) + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + for name, cls in inspect.getmembers(module, inspect.isclass): + if issubclass(cls, DirectionalStrategyBase) and cls is not DirectionalStrategyBase: + strategy_classes.append(cls) + return strategy_classes + +strategies_path = '/path/to/your/strategies' # update this to the path to your strategies +strategy_classes = load_strategies(strategies_path) +for strategy_cls in strategy_classes: + print(f"Class: {strategy_cls.__name__}") + print("Parameters: ", inspect.signature(strategy_cls).parameters) From 970339911843e2039538c499c819998614955029 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 14:54:14 +0200 Subject: [PATCH 41/64] (feat) add directional strategy template --- utils/file_templates.py | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 utils/file_templates.py diff --git a/utils/file_templates.py b/utils/file_templates.py new file mode 100644 index 0000000..2dc2599 --- /dev/null +++ b/utils/file_templates.py @@ -0,0 +1,57 @@ +def directional_strategy_template(strategy_name: str, + exchange: str, + trading_pair: str, + interval: str) -> str: + return f"""import pandas_ta as ta +import pandas as pd +import numpy as np + +from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase +from quants_lab.utils import data_management + + + +class {strategy_name}(DirectionalStrategyBase): + # Define the attributes of the strategy + def __init__(self, + exchange="{exchange}", + trading_pair="{trading_pair}", + interval="{interval}"): + self.exchange = exchange + self.trading_pair = trading_pair + self.interval = interval + + def get_raw_data(self): + # The method get candles will search for the data in the folder data/candles + # If the data is not there, you can use the candles downloader to get the data + df = self.get_candles( + exchange=self.exchange, + trading_pair=self.trading_pair, + interval=self.interval, + ) + return df + + def add_indicators(self, df): + df.ta.sma(length=20, append=True) + # ... Add more indicators here + # ... Check https://github.com/twopirllc/pandas-ta#indicators-by-category for more indicators + # ... Use help(ta.indicator_name) to get more info + return df + + def add_signals(self, df): + # ... Do your own logic + random_series = pd.Series(np.random.randint(low=0, high=101, size=100)) + random_series_2 = pd.Series(np.random.randint(low=0, high=101, size=100)) + random_thold = np.random.randint(low=45, high=65) + random_thold_2 = np.random.randint(low=45, high=65) + + # Generate long and short conditions + macd_long_cond = (random_series > random_thold) & (random_series_2 > random_thold_2) + macd_short_cond = (random_series < random_thold) & (random_series_2 > random_thold_2) + + # Choose side + df['side'] = 0 + df.loc[macd_long_cond, 'side'] = 1 + df.loc[macd_short_cond, 'side'] = -1 + return df +""" \ No newline at end of file From 437613ab5569363610e1238ca806273073cc0486 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 14:54:34 +0200 Subject: [PATCH 42/64] (feat) initialize backtesting page --- pages/9_⚙️_Backtesting.py | 45 ++++++++++------------------------------ 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py index 39b1136..898054d 100644 --- a/pages/9_⚙️_Backtesting.py +++ b/pages/9_⚙️_Backtesting.py @@ -1,12 +1,5 @@ -import pandas as pd -import pandas_ta as ta import streamlit as st -from quants_lab.strategy.mean_reversion.bollinger import Bollinger -from quants_lab.strategy.mean_reversion.stat_arb import StatArb -from quants_lab.utils import data_management -from quants_lab.backtesting.backtesting import Backtesting -from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis st.set_page_config( page_title="Hummingbot Dashboard", @@ -16,35 +9,19 @@ st.set_page_config( ) st.title("⚙️ Backtesting") -df_to_show = data_management.get_dataframe( - exchange='binance_perpetual', - trading_pair="ETH-USDT", - interval='1h', -) +create, modify, backtest, optimize, analyze = st.tabs(["Create", "Modify", "Backtest", "Optimize", "Analyze"]) +with create: + pass -strategy = StatArb(trading_pair="ETH-USDT", periods=24, deviation_threshold=1.5) +with modify: + pass -backtesting = Backtesting(strategy=strategy) +with backtest: + pass -positions = backtesting.run_backtesting( - # start='2022-01-01', - # end='2023-06-02', - order_amount=50, - leverage=20, - initial_portfolio=100, - take_profit_multiplier=3.0, - stop_loss_multiplier=1.5, - time_limit=60 * 60 * 24, - std_span=None, -) -backtesting_analysis = BacktestingAnalysis(positions=positions, candles_df=df_to_show) -backtesting_analysis.create_base_figure(volume=False, positions=True, extra_rows=1) -backtesting_analysis.add_trade_pnl(row=2) - -c1, c2 = st.columns([0.2, 0.8]) -with c1: - st.text(backtesting_analysis.text_report()) -with c2: - st.plotly_chart(backtesting_analysis.figure(), use_container_width=True) +with optimize: + pass +with analyze: + pass From f81579fdcacf009d603314df5d9c7c87e3da8d4a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 15:47:18 +0200 Subject: [PATCH 43/64] (feat) remove strategy optimizer --- quants_lab/strategy/strategy_optimizer.py | 77 ----------------------- 1 file changed, 77 deletions(-) delete mode 100644 quants_lab/strategy/strategy_optimizer.py diff --git a/quants_lab/strategy/strategy_optimizer.py b/quants_lab/strategy/strategy_optimizer.py deleted file mode 100644 index bfbf441..0000000 --- a/quants_lab/strategy/strategy_optimizer.py +++ /dev/null @@ -1,77 +0,0 @@ -import optuna -from quants_lab.backtesting.backtesting import Backtesting -from quants_lab.backtesting.backtesting_analysis import BacktestingAnalysis -from quants_lab.strategy.mean_reversion.bollinger import Bollinger -from quants_lab.strategy.mean_reversion.macd_bb import MACDBB -from optuna.exceptions import TrialPruned - -from quants_lab.strategy.mean_reversion.stat_arb import StatArb - -STUDY_NAME = "stat_arb" - - -def objective(trial): - # strategy = Bollinger( - # exchange="binance_perpetual", - # trading_pair="ETH-USDT", - # interval="3m", - # bb_length=trial.suggest_int("bb_length", 20, 300), - # bb_std=trial.suggest_float("bb_std", 1.0, 3.0), - # bb_long_threshold=trial.suggest_float("bb_long_threshold", -0.5, 0.3), - # bb_short_threshold=trial.suggest_float("bb_short_threshold", 0.7, 1.5), - # ) - - # fast_macd = trial.suggest_int("fast_macd", 10, 50) - # strategy = MACDBB( - # exchange="binance_perpetual", - # trading_pair="ETH-USDT", - # interval="3m", - # bb_length=trial.suggest_int("bb_length", 20, 300), - # bb_std=trial.suggest_float("bb_std", 1.0, 3.0), - # bb_long_threshold=trial.suggest_float("bb_long_threshold", -0.5, 0.3), - # bb_short_threshold=trial.suggest_float("bb_short_threshold", 0.7, 1.5), - # fast_macd=fast_macd, - # slow_macd=trial.suggest_int("slow_macd", fast_macd + 1, 100), - # signal_macd=trial.suggest_int("signal_macd", 8, 54) - # ) - strategy = StatArb(trading_pair="ETH-USDT", - periods=trial.suggest_int("periods", 10, 150), - deviation_threshold=trial.suggest_float("deviation_threshold", 0.9, 2.0)) - try: - backtesting = Backtesting(strategy=strategy) - backtesting_result = backtesting.run_backtesting( - start='2021-04-01', - order_amount=50, - leverage=20, - initial_portfolio=100, - take_profit_multiplier=trial.suggest_float("take_profit_multiplier", 1.0, 3.0), - stop_loss_multiplier=trial.suggest_float("stop_loss_multiplier", 1.0, 3.0), - time_limit=60 * 60 * trial.suggest_int("time_limit", 1, 24), - std_span=None, - ) - backtesting_analysis = BacktestingAnalysis( - positions=backtesting_result, - ) - - trial.set_user_attr("net_profit_usd", backtesting_analysis.net_profit_usd()) - trial.set_user_attr("net_profit_pct", backtesting_analysis.net_profit_pct()) - trial.set_user_attr("max_drawdown_usd", backtesting_analysis.max_drawdown_usd()) - trial.set_user_attr("max_drawdown_pct", backtesting_analysis.max_drawdown_pct()) - trial.set_user_attr("sharpe_ratio", backtesting_analysis.sharpe_ratio()) - trial.set_user_attr("accuracy", backtesting_analysis.accuracy()) - trial.set_user_attr("total_positions", backtesting_analysis.total_positions()) - trial.set_user_attr("win_signals", backtesting_analysis.win_signals().shape[0]) - trial.set_user_attr("loss_signals", backtesting_analysis.loss_signals().shape[0]) - trial.set_user_attr("profit_factor", backtesting_analysis.profit_factor()) - trial.set_user_attr("duration_in_hours", backtesting_analysis.duration_in_minutes() / 60) - trial.set_user_attr("avg_trading_time_in_hours", backtesting_analysis.avg_trading_time_in_minutes() / 60) - return backtesting_analysis.net_profit_pct() - except Exception as e: - print(e) - raise TrialPruned() - - -study = optuna.create_study(direction="maximize", study_name=STUDY_NAME, storage="sqlite:///backtesting_report.db", - load_if_exists=True) - -study.optimize(objective, n_trials=2000) From 43784ccd461d050df7827e41bbc391ff056a66e8 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 15:47:26 +0200 Subject: [PATCH 44/64] (feat) rename class --- quants_lab/strategy/strategy_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quants_lab/strategy/strategy_analysis.py b/quants_lab/strategy/strategy_analysis.py index 095e907..b93f73b 100644 --- a/quants_lab/strategy/strategy_analysis.py +++ b/quants_lab/strategy/strategy_analysis.py @@ -7,7 +7,7 @@ import plotly.graph_objs as go import numpy as np -class BacktestingAnalysis: +class StrategyAnalysis: def __init__(self, positions: pd.DataFrame, candles_df: Optional[pd.DataFrame] = None): self.candles_df = candles_df self.positions = positions From 2dc689b4a2ac176ea4078429ca891b6cde938be2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 15:48:35 +0200 Subject: [PATCH 45/64] (feat) refactor strategy with pydantic --- quants_lab/strategy/experiments/stat_arb.py | 46 ++++++++++++--------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/quants_lab/strategy/experiments/stat_arb.py b/quants_lab/strategy/experiments/stat_arb.py index 8ec47fd..d528bd0 100644 --- a/quants_lab/strategy/experiments/stat_arb.py +++ b/quants_lab/strategy/experiments/stat_arb.py @@ -1,32 +1,38 @@ import pandas as pd import pandas_ta as ta +from pydantic import BaseModel, Field + from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase -from quants_lab.utils import data_management + +class StatArbConfig(BaseModel): + exchange: str = Field(default="binance_perpetual") + trading_pair: str = Field(default="ETH-USDT") + target_trading_pair: str = Field(default="BTC-USDT") + interval: str = Field(default="1h") + lookback: int = Field(default=100, ge=2, le=10000) + z_score_long: float = Field(default=2, ge=0, le=5) + z_score_short: float = Field(default=-2, ge=-5, le=0) class StatArb(DirectionalStrategyBase): - def __init__(self, - exchange="binance_perpetual", - trading_pair="DOGE-USDT", - target_trading_pair="BTC-USDT", - interval="1h", - periods=100, - deviation_threshold=1.1): - self.exchange = exchange - self.trading_pair = trading_pair - self.interval = interval - self.target_trading_pair = target_trading_pair - self.periods = periods - self.deviation_threshold = deviation_threshold + def __init__(self, config: StatArbConfig): + super().__init__(config) + self.exchange = config.exchange + self.trading_pair = config.trading_pair + self.target_trading_pair = config.target_trading_pair + self.interval = config.interval + self.lookback = config.lookback + self.z_score_long = config.z_score_long + self.z_score_short = config.z_score_short def get_raw_data(self): - df = data_management.get_dataframe( + df = self.get_candles( exchange=self.exchange, trading_pair=self.trading_pair, interval=self.interval, ) - df_target = data_management.get_dataframe( + df_target = self.get_candles( exchange=self.exchange, trading_pair=self.target_trading_pair, interval=self.interval, @@ -38,14 +44,14 @@ class StatArb(DirectionalStrategyBase): df["pct_change_original"] = df["close"].pct_change() df["pct_change_target"] = df["close_target"].pct_change() df["spread"] = df["pct_change_target"] - df["pct_change_original"] - df["cum_spread"] = df["spread"].rolling(self.periods).sum() - df["z_score"] = ta.zscore(df["cum_spread"], length=self.periods) + df["cum_spread"] = df["spread"].rolling(self.lookback).sum() + df["z_score"] = ta.zscore(df["cum_spread"], length=self.lookback) return df def predict(self, df): df["side"] = 0 - short_condition = df["z_score"] < - self.deviation_threshold - long_condition = df["z_score"] > self.deviation_threshold + short_condition = df["z_score"] < - self.z_score_short + long_condition = df["z_score"] > self.z_score_long df.loc[long_condition, "side"] = 1 df.loc[short_condition, "side"] = -1 return df From 73b8d1613a299cbdad2cdbc5db36ef9fa083495c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 15:48:43 +0200 Subject: [PATCH 46/64] (feat) rename function --- utils/os_utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/utils/os_utils.py b/utils/os_utils.py index 863d75f..84f38c7 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -57,7 +57,7 @@ def get_python_files_from_directory(directory: str) -> list: return py_files -def load_strategies(path): +def load_directional_strategies(path): strategy_classes = [] for filename in os.listdir(path): if filename.endswith('.py'): @@ -71,8 +71,3 @@ def load_strategies(path): strategy_classes.append(cls) return strategy_classes -strategies_path = '/path/to/your/strategies' # update this to the path to your strategies -strategy_classes = load_strategies(strategies_path) -for strategy_cls in strategy_classes: - print(f"Class: {strategy_cls.__name__}") - print("Parameters: ", inspect.signature(strategy_cls).parameters) From 3447d650ef7aea6174331df81aa0aa609e2b1bc6 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 15:48:59 +0200 Subject: [PATCH 47/64] (feat) add base pydantic config --- quants_lab/strategy/directional_strategy_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index b75d049..afe2502 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -2,12 +2,16 @@ import os from datetime import datetime from typing import Optional import pandas as pd +from pydantic import BaseModel from quants_lab.labeling.triple_barrier_method import triple_barrier_method class DirectionalStrategyBase: + def __init__(self, config: BaseModel): + self.config = config + def get_data(self, start: Optional[str] = None, end: Optional[str] = None): df = self.get_raw_data() return self.filter_df_by_time(df, start, end) From f8be76778eefaf32b52b14370ecd96d55b19b45e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 15:49:14 +0200 Subject: [PATCH 48/64] (feat) refactor bollinger strategy --- quants_lab/strategy/experiments/bollinger.py | 38 +++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/quants_lab/strategy/experiments/bollinger.py b/quants_lab/strategy/experiments/bollinger.py index af252b8..b5fa81f 100644 --- a/quants_lab/strategy/experiments/bollinger.py +++ b/quants_lab/strategy/experiments/bollinger.py @@ -1,28 +1,32 @@ import pandas_ta as ta +from pydantic import BaseModel, Field + from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase -from quants_lab.utils import data_management + +class BollingerConf(BaseModel): + exchange: str = Field(default="binance_perpetual") + trading_pair: str = Field(default="ETH-USDT") + interval: str = Field(default="1h") + bb_length: int = Field(default=100, ge=2, le=1000) + bb_std: float = Field(default=2.0, ge=0.5, le=4.0) + bb_long_threshold: float = Field(default=0.0, ge=-3.0, le=0.5) + bb_short_threshold: float = Field(default=1.0, ge=0.5, le=3.0) class Bollinger(DirectionalStrategyBase): - def __init__(self, - exchange="binance_perpetual", - trading_pair="ETH-USDT", - interval="1h", - bb_length=24, - bb_std=2.0, - bb_long_threshold=0.0, - bb_short_threshold=1.0,): - self.exchange = exchange - self.trading_pair = trading_pair - self.interval = interval - self.bb_length = bb_length - self.bb_std = bb_std - self.bb_long_threshold = bb_long_threshold - self.bb_short_threshold = bb_short_threshold + def __init__(self, config: BollingerConf): + super().__init__(config) + self.exchange = config.exchange + self.trading_pair = config.trading_pair + self.interval = config.interval + self.bb_length = config.bb_length + self.bb_std = config.bb_std + self.bb_long_threshold = config.bb_long_threshold + self.bb_short_threshold = config.bb_short_threshold def get_raw_data(self): - df = data_management.get_dataframe( + df = self.get_candles( exchange=self.exchange, trading_pair=self.trading_pair, interval=self.interval, From 61a60d133d7625a3825de0ca29a368c04295d74d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 15:49:24 +0200 Subject: [PATCH 49/64] (feat) add strategies path --- constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/constants.py b/constants.py index 297706a..5779651 100644 --- a/constants.py +++ b/constants.py @@ -1,3 +1,4 @@ CANDLES_DATA_PATH = "data/candles" DOWNLOAD_CANDLES_CONFIG_YML = "hummingbot_files/scripts_configs/data_downloader_config.yml" BOTS_FOLDER = "hummingbot_files/bot_configs" +DIRECTIONAL_STRATEGIES_PATH = "quants_lab/strategy/experiments" From 00db641a6732484ee3f979338faf1b3aadee39a8 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 16:02:55 +0200 Subject: [PATCH 50/64] (feat) final refactor of strategies configs --- .../strategy/directional_strategy_base.py | 3 +- quants_lab/strategy/experiments/bollinger.py | 22 ++----- quants_lab/strategy/experiments/macd_bb.py | 61 ++++++++----------- quants_lab/strategy/experiments/stat_arb.py | 30 +++------ 4 files changed, 43 insertions(+), 73 deletions(-) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index afe2502..8c9b71d 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -8,8 +8,7 @@ from quants_lab.labeling.triple_barrier_method import triple_barrier_method class DirectionalStrategyBase: - - def __init__(self, config: BaseModel): + def __init__(self, config): self.config = config def get_data(self, start: Optional[str] = None, end: Optional[str] = None): diff --git a/quants_lab/strategy/experiments/bollinger.py b/quants_lab/strategy/experiments/bollinger.py index b5fa81f..2a49867 100644 --- a/quants_lab/strategy/experiments/bollinger.py +++ b/quants_lab/strategy/experiments/bollinger.py @@ -15,32 +15,22 @@ class BollingerConf(BaseModel): class Bollinger(DirectionalStrategyBase): - def __init__(self, config: BollingerConf): - super().__init__(config) - self.exchange = config.exchange - self.trading_pair = config.trading_pair - self.interval = config.interval - self.bb_length = config.bb_length - self.bb_std = config.bb_std - self.bb_long_threshold = config.bb_long_threshold - self.bb_short_threshold = config.bb_short_threshold - def get_raw_data(self): df = self.get_candles( - exchange=self.exchange, - trading_pair=self.trading_pair, - interval=self.interval, + exchange=self.config.exchange, + trading_pair=self.config.trading_pair, + interval=self.config.interval, ) return df def preprocessing(self, df): - df.ta.bbands(length=self.bb_length, std=self.bb_std, append=True) + df.ta.bbands(length=self.config.bb_length, std=self.config.bb_std, append=True) return df def predict(self, df): df["side"] = 0 - long_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] < self.bb_long_threshold - short_condition = df[f"BBP_{self.bb_length}_{self.bb_std}"] > self.bb_short_threshold + long_condition = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] < self.config.bb_long_threshold + short_condition = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] > self.config.bb_short_threshold df.loc[long_condition, "side"] = 1 df.loc[short_condition, "side"] = -1 return df diff --git a/quants_lab/strategy/experiments/macd_bb.py b/quants_lab/strategy/experiments/macd_bb.py index f6712d6..813e111 100644 --- a/quants_lab/strategy/experiments/macd_bb.py +++ b/quants_lab/strategy/experiments/macd_bb.py @@ -1,52 +1,43 @@ import pandas_ta as ta +from pydantic import BaseModel, Field + from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase -from quants_lab.utils import data_management + +class MACDBBConfig(BaseModel): + exchange: str = Field(default="binance_perpetual") + trading_pair: str = Field(default="ETH-USDT") + interval: str = Field(default="1h") + bb_length: int = Field(default=24, ge=2, le=1000) + bb_std: float = Field(default=2.0, ge=0.5, le=4.0) + bb_long_threshold: float = Field(default=0.0, ge=-3.0, le=0.5) + bb_short_threshold: float = Field(default=1.0, ge=0.5, le=3.0) + fast_macd: int = Field(default=21, ge=2, le=100) + slow_macd: int = Field(default=42, ge=fast_macd, le=1000) + signal_macd: int = Field(default=9, ge=2, le=100) -class MACDBB(DirectionalStrategyBase): - def __init__(self, - exchange="binance_perpetual", - trading_pair="ETH-USDT", - interval="1h", - bb_length=24, - bb_std=2.0, - bb_long_threshold=0.05, - bb_short_threshold=0.95, - fast_macd=21, - slow_macd=42, - signal_macd=9): - self.exchange = exchange - self.trading_pair = trading_pair - self.interval = interval - self.bb_length = bb_length - self.bb_std = bb_std - self.bb_long_threshold = bb_long_threshold - self.bb_short_threshold = bb_short_threshold - self.fast_macd = fast_macd - self.slow_macd = slow_macd - self.signal_macd = signal_macd - +class MacdBollinger(DirectionalStrategyBase): def get_raw_data(self): - df = data_management.get_dataframe( - exchange=self.exchange, - trading_pair=self.trading_pair, - interval=self.interval, + df = self.get_candles( + exchange=self.config.exchange, + trading_pair=self.config.trading_pair, + interval=self.config.interval, ) return df def preprocessing(self, df): - df.ta.bbands(length=self.bb_length, std=self.bb_std, append=True) - df.ta.macd(fast=self.fast_macd, slow=self.slow_macd, signal=self.signal_macd, append=True) + df.ta.bbands(length=self.config.bb_length, std=self.config.bb_std, append=True) + df.ta.macd(fast=self.config.fast_macd, slow=self.config.slow_macd, signal=self.config.signal_macd, append=True) return df def predict(self, df): - bbp = df[f"BBP_{self.bb_length}_{self.bb_std}"] - macdh = df[f"MACDh_{self.fast_macd}_{self.slow_macd}_{self.signal_macd}"] - macd = df[f"MACD_{self.fast_macd}_{self.slow_macd}_{self.signal_macd}"] + bbp = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] + macdh = df[f"MACDh_{self.config.fast_macd}_{self.config.slow_macd}_{self.config.signal_macd}"] + macd = df[f"MACD_{self.config.fast_macd}_{self.config.slow_macd}_{self.config.signal_macd}"] - long_condition = (bbp < self.bb_long_threshold) & (macdh > 0) & (macd < 0) - short_condition = (bbp > self.bb_short_threshold) & (macdh < 0) & (macd > 0) + long_condition = (bbp < self.config.bb_long_threshold) & (macdh > 0) & (macd < 0) + short_condition = (bbp > self.config.bb_short_threshold) & (macdh < 0) & (macd > 0) df["side"] = 0 df.loc[long_condition, "side"] = 1 diff --git a/quants_lab/strategy/experiments/stat_arb.py b/quants_lab/strategy/experiments/stat_arb.py index d528bd0..51f73d4 100644 --- a/quants_lab/strategy/experiments/stat_arb.py +++ b/quants_lab/strategy/experiments/stat_arb.py @@ -16,26 +16,16 @@ class StatArbConfig(BaseModel): class StatArb(DirectionalStrategyBase): - def __init__(self, config: StatArbConfig): - super().__init__(config) - self.exchange = config.exchange - self.trading_pair = config.trading_pair - self.target_trading_pair = config.target_trading_pair - self.interval = config.interval - self.lookback = config.lookback - self.z_score_long = config.z_score_long - self.z_score_short = config.z_score_short - def get_raw_data(self): df = self.get_candles( - exchange=self.exchange, - trading_pair=self.trading_pair, - interval=self.interval, + exchange=self.config.exchange, + trading_pair=self.config.trading_pair, + interval=self.config.interval, ) df_target = self.get_candles( - exchange=self.exchange, - trading_pair=self.target_trading_pair, - interval=self.interval, + exchange=self.config.exchange, + trading_pair=self.config.target_trading_pair, + interval=self.config.interval, ) df = pd.merge(df, df_target, on="timestamp", how='inner', suffixes=('', '_target')) return df @@ -44,14 +34,14 @@ class StatArb(DirectionalStrategyBase): df["pct_change_original"] = df["close"].pct_change() df["pct_change_target"] = df["close_target"].pct_change() df["spread"] = df["pct_change_target"] - df["pct_change_original"] - df["cum_spread"] = df["spread"].rolling(self.lookback).sum() - df["z_score"] = ta.zscore(df["cum_spread"], length=self.lookback) + df["cum_spread"] = df["spread"].rolling(self.config.lookback).sum() + df["z_score"] = ta.zscore(df["cum_spread"], length=self.config.lookback) return df def predict(self, df): df["side"] = 0 - short_condition = df["z_score"] < - self.z_score_short - long_condition = df["z_score"] > self.z_score_long + short_condition = df["z_score"] < - self.config.z_score_short + long_condition = df["z_score"] > self.config.z_score_long df.loc[long_condition, "side"] = 1 df.loc[short_condition, "side"] = -1 return df From 35a63e098c020fe7ab6a1cddca4b4f16950385b4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:31:39 +0200 Subject: [PATCH 51/64] (feat) update conda environment --- environment_conda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment_conda.yml b/environment_conda.yml index 4d9cdcb..82286d9 100644 --- a/environment_conda.yml +++ b/environment_conda.yml @@ -19,5 +19,6 @@ dependencies: - jupyter - optuna - optuna-dashboard + - streamlit-ace - git+https://github.com/hummingbot/hbot-remote-client-py.git - git+https://github.com/hummingbot/docker-manager.git From f952d629590e6ed59ac7e9b0f9fcfc193afedd28 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:32:29 +0200 Subject: [PATCH 52/64] (feat) add modules management functionalities --- utils/os_utils.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/utils/os_utils.py b/utils/os_utils.py index 84f38c7..9918ab7 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -3,6 +3,9 @@ import subprocess import importlib.util import inspect import os + +from pydantic import BaseModel + from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase # update this to the actual import import yaml @@ -58,16 +61,35 @@ def get_python_files_from_directory(directory: str) -> list: def load_directional_strategies(path): - strategy_classes = [] + strategies = {} for filename in os.listdir(path): - if filename.endswith('.py'): + if filename.endswith('.py') and "__init__" not in filename: module_name = filename[:-3] # strip the .py to get the module name + strategies[module_name] = {"module": module_name} file_path = os.path.join(path, filename) spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) for name, cls in inspect.getmembers(module, inspect.isclass): if issubclass(cls, DirectionalStrategyBase) and cls is not DirectionalStrategyBase: - strategy_classes.append(cls) - return strategy_classes + strategies[module_name]["class"] = cls + if issubclass(cls, BaseModel) and cls is not BaseModel: + strategies[module_name]["config"] = cls + return strategies + +def get_function_from_file(file_path: str, function_name: str): + # Create a module specification from the file path and load it + spec = importlib.util.spec_from_file_location("module.name", file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Get the function from the module + function = getattr(module, function_name) + return function + + +def execute_bash_command(command: str, shell: bool = True, wait: bool = False): + process = subprocess.Popen(command, shell=shell) + if wait: + process.wait() From 087b0704cc0dbfd5df629c931304882cadaaca96 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:37:08 +0200 Subject: [PATCH 53/64] (feat) add trade pnl --- quants_lab/strategy/strategy_analysis.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/quants_lab/strategy/strategy_analysis.py b/quants_lab/strategy/strategy_analysis.py index b93f73b..55c3a3c 100644 --- a/quants_lab/strategy/strategy_analysis.py +++ b/quants_lab/strategy/strategy_analysis.py @@ -13,8 +13,8 @@ class StrategyAnalysis: self.positions = positions self.base_figure = None - def create_base_figure(self, candlestick=True, volume=True, positions=False, extra_rows=1): - rows, heights = self.get_n_rows_and_heights(extra_rows, volume) + def create_base_figure(self, candlestick=True, volume=True, positions=False, trade_pnl=False, extra_rows=0): + rows, heights = self.get_n_rows_and_heights(extra_rows + volume + trade_pnl, volume) self.rows = rows specs = [[{"secondary_y": True}]] * rows self.base_figure = make_subplots(rows=rows, cols=1, shared_xaxes=True, vertical_spacing=0.05, @@ -25,6 +25,8 @@ class StrategyAnalysis: self.add_volume() if positions: self.add_positions() + if trade_pnl: + self.add_trade_pnl() self.update_layout(volume) def add_positions(self): @@ -101,7 +103,7 @@ class StrategyAnalysis: row=2, col=1, ) - def add_trade_pnl(self, row=4): + def add_trade_pnl(self, row=2): self.base_figure.add_trace( go.Scatter( x=self.positions['timestamp'], From c5e42dfb50250e2f430f815512be618f162d0345 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:37:26 +0200 Subject: [PATCH 54/64] (feat) init optimizations --- quants_lab/optimizations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 quants_lab/optimizations/__init__.py diff --git a/quants_lab/optimizations/__init__.py b/quants_lab/optimizations/__init__.py new file mode 100644 index 0000000..e69de29 From a3f0a19a5dd00c612f3b4f0d9aad78e7c3ed098e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:37:42 +0200 Subject: [PATCH 55/64] (feat) update strategies config --- quants_lab/strategy/directional_strategy_base.py | 8 +++++--- quants_lab/strategy/experiments/bollinger.py | 2 +- quants_lab/strategy/experiments/macd_bb.py | 4 ++-- quants_lab/strategy/experiments/stat_arb.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index 8c9b71d..cb623d8 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -1,14 +1,16 @@ import os from datetime import datetime -from typing import Optional +from typing import Optional, TypeVar, Generic import pandas as pd from pydantic import BaseModel from quants_lab.labeling.triple_barrier_method import triple_barrier_method +ConfigType = TypeVar("ConfigType", bound=BaseModel) -class DirectionalStrategyBase: - def __init__(self, config): + +class DirectionalStrategyBase(Generic[ConfigType]): + def __init__(self, config: ConfigType): self.config = config def get_data(self, start: Optional[str] = None, end: Optional[str] = None): diff --git a/quants_lab/strategy/experiments/bollinger.py b/quants_lab/strategy/experiments/bollinger.py index 2a49867..9dd374e 100644 --- a/quants_lab/strategy/experiments/bollinger.py +++ b/quants_lab/strategy/experiments/bollinger.py @@ -14,7 +14,7 @@ class BollingerConf(BaseModel): bb_short_threshold: float = Field(default=1.0, ge=0.5, le=3.0) -class Bollinger(DirectionalStrategyBase): +class Bollinger(DirectionalStrategyBase[BollingerConf]): def get_raw_data(self): df = self.get_candles( exchange=self.config.exchange, diff --git a/quants_lab/strategy/experiments/macd_bb.py b/quants_lab/strategy/experiments/macd_bb.py index 813e111..a2ba82c 100644 --- a/quants_lab/strategy/experiments/macd_bb.py +++ b/quants_lab/strategy/experiments/macd_bb.py @@ -13,11 +13,11 @@ class MACDBBConfig(BaseModel): bb_long_threshold: float = Field(default=0.0, ge=-3.0, le=0.5) bb_short_threshold: float = Field(default=1.0, ge=0.5, le=3.0) fast_macd: int = Field(default=21, ge=2, le=100) - slow_macd: int = Field(default=42, ge=fast_macd, le=1000) + slow_macd: int = Field(default=42, ge=30, le=1000) signal_macd: int = Field(default=9, ge=2, le=100) -class MacdBollinger(DirectionalStrategyBase): +class MacdBollinger(DirectionalStrategyBase[MACDBBConfig]): def get_raw_data(self): df = self.get_candles( exchange=self.config.exchange, diff --git a/quants_lab/strategy/experiments/stat_arb.py b/quants_lab/strategy/experiments/stat_arb.py index 51f73d4..a9060d4 100644 --- a/quants_lab/strategy/experiments/stat_arb.py +++ b/quants_lab/strategy/experiments/stat_arb.py @@ -15,7 +15,7 @@ class StatArbConfig(BaseModel): z_score_short: float = Field(default=-2, ge=-5, le=0) -class StatArb(DirectionalStrategyBase): +class StatArb(DirectionalStrategyBase[StatArbConfig]): def get_raw_data(self): df = self.get_candles( exchange=self.config.exchange, From b6ef8ccce0d2de763b5bfa857872e82f81fc6c8a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:37:48 +0200 Subject: [PATCH 56/64] (feat) add more constants --- constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/constants.py b/constants.py index 5779651..dcc0d45 100644 --- a/constants.py +++ b/constants.py @@ -2,3 +2,4 @@ CANDLES_DATA_PATH = "data/candles" DOWNLOAD_CANDLES_CONFIG_YML = "hummingbot_files/scripts_configs/data_downloader_config.yml" BOTS_FOLDER = "hummingbot_files/bot_configs" DIRECTIONAL_STRATEGIES_PATH = "quants_lab/strategy/experiments" +OPTIMIZATIONS_PATH = "quants_lab/optimizations" From bf483974732164f750c4a7a49f45a3c02b691d36 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:38:02 +0200 Subject: [PATCH 57/64] (feat) add file template for optimization --- utils/file_templates.py | 76 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/utils/file_templates.py b/utils/file_templates.py index 2dc2599..df0a062 100644 --- a/utils/file_templates.py +++ b/utils/file_templates.py @@ -1,3 +1,6 @@ +from typing import Dict + + def directional_strategy_template(strategy_name: str, exchange: str, trading_pair: str, @@ -7,7 +10,6 @@ import pandas as pd import numpy as np from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase -from quants_lab.utils import data_management @@ -54,4 +56,74 @@ class {strategy_name}(DirectionalStrategyBase): df.loc[macd_long_cond, 'side'] = 1 df.loc[macd_short_cond, 'side'] = -1 return df -""" \ No newline at end of file +""" + + +def get_optuna_suggest_str(field_name: str, properties: Dict): + map_by_type = { + "number": "trial.suggest_float", + "integer": "trial.suggest_int", + "string": "trial.suggest_categorical", + } + config_num = f"('{field_name}', {properties.get('minimum', '_')}, {properties.get('maximum', '_')})" + config_cat = f"('{field_name}', ['{properties.get('default', '_')}', '_'])" + optuna_trial_str = map_by_type[properties["type"]] + config_num if properties["type"] != "string" \ + else map_by_type[properties["type"]] + config_cat + + return f"{field_name}={optuna_trial_str}" + + +def strategy_optimization_template(strategy_info: dict): + strategy_cls = strategy_info["class"] + strategy_config = strategy_info["config"] + strategy_module = strategy_info["module"] + field_schema = strategy_config.schema()["properties"] + fields_str = [get_optuna_suggest_str(field_name, properties) for field_name, properties in field_schema.items()] + fields_str = "".join([f" {field_str},\n" for field_str in fields_str]) + return f""" +import traceback + +from optuna import TrialPruned + +from quants_lab.strategy.experiments.{strategy_module} import {strategy_cls.__name__}, {strategy_config.__name__} +from quants_lab.strategy.strategy_analysis import StrategyAnalysis + + +def objective(trial): + try: + config = {strategy_config.__name__}( +{fields_str} + ) + strategy = {strategy_cls.__name__}(config=config) + market_data, positions = strategy.run_backtesting( + start='2021-04-01', + order_amount=50, + leverage=20, + initial_portfolio=100, + take_profit_multiplier=trial.suggest_float("take_profit_multiplier", 1.0, 3.0), + stop_loss_multiplier=trial.suggest_float("stop_loss_multiplier", 1.0, 3.0), + time_limit=60 * 60 * trial.suggest_int("time_limit", 1, 24), + std_span=None, + ) + strategy_analysis = StrategyAnalysis( + positions=positions, + ) + + trial.set_user_attr("net_profit_usd", strategy_analysis.net_profit_usd()) + trial.set_user_attr("net_profit_pct", strategy_analysis.net_profit_pct()) + trial.set_user_attr("max_drawdown_usd", strategy_analysis.max_drawdown_usd()) + trial.set_user_attr("max_drawdown_pct", strategy_analysis.max_drawdown_pct()) + trial.set_user_attr("sharpe_ratio", strategy_analysis.sharpe_ratio()) + trial.set_user_attr("accuracy", strategy_analysis.accuracy()) + trial.set_user_attr("total_positions", strategy_analysis.total_positions()) + trial.set_user_attr("win_signals", strategy_analysis.win_signals().shape[0]) + trial.set_user_attr("loss_signals", strategy_analysis.loss_signals().shape[0]) + trial.set_user_attr("profit_factor", strategy_analysis.profit_factor()) + trial.set_user_attr("duration_in_hours", strategy_analysis.duration_in_minutes() / 60) + trial.set_user_attr("avg_trading_time_in_hours", strategy_analysis.avg_trading_time_in_minutes() / 60) + return strategy_analysis.net_profit_pct() + except Exception as e: + # TODO: Log error + traceback.print_exc() + raise TrialPruned() + """ From 2da66f5275d6bb4ca53ed940f9639186af249f5b Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Jul 2023 21:38:10 +0200 Subject: [PATCH 58/64] (feat) MVP of backtest and optimize tab --- pages/9_⚙️_Backtesting.py | 126 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 4 deletions(-) diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py index 898054d..c335b06 100644 --- a/pages/9_⚙️_Backtesting.py +++ b/pages/9_⚙️_Backtesting.py @@ -1,12 +1,25 @@ -import streamlit as st +import datetime +import threading +import webbrowser +import streamlit as st +from streamlit_ace import st_ace + +import constants +from quants_lab.strategy.strategy_analysis import StrategyAnalysis +from utils import os_utils +from utils.file_templates import strategy_optimization_template +from utils.os_utils import load_directional_strategies, save_file, get_function_from_file +import optuna st.set_page_config( page_title="Hummingbot Dashboard", page_icon="🚀", layout="wide", - initial_sidebar_state="collapsed" ) +if "strategy_params" not in st.session_state: + st.session_state.strategy_params = {} + st.title("⚙️ Backtesting") create, modify, backtest, optimize, analyze = st.tabs(["Create", "Modify", "Backtest", "Optimize", "Analyze"]) @@ -18,10 +31,115 @@ with modify: pass with backtest: - pass + # TODO: + # * Add videos explaining how to the triple barrier method works and how the backtesting is designed, + # link to video of how to create a strategy, etc in a toggle. + # * Add performance analysis graphs of the backtesting run + strategies = load_directional_strategies(constants.DIRECTIONAL_STRATEGIES_PATH) + strategy_to_optimize = st.selectbox("Select strategy to backtest", strategies.keys()) + strategy = strategies[strategy_to_optimize] + strategy_config = strategy["config"] + field_schema = strategy_config.schema()["properties"] + st.write("## Strategy parameters") + c1, c2 = st.columns([5, 1]) + with c1: + columns = st.columns(4) + column_index = 0 + for field_name, properties in field_schema.items(): + field_type = properties["type"] + with columns[column_index]: + if field_type in ["number", "integer"]: + field_value = st.number_input(field_name, + value=properties["default"], + min_value=properties.get("minimum"), + max_value=properties.get("maximum"), + key=field_name) + elif field_type == "string": + field_value = st.text_input(field_name, value=properties["default"]) + elif field_type == "boolean": + # TODO: Add support for boolean fields in optimize tab + field_value = st.checkbox(field_name, value=properties["default"]) + else: + raise ValueError(f"Field type {field_type} not supported") + st.session_state["strategy_params"][field_name] = field_value + column_index = (column_index + 1) % 4 + with c2: + add_positions = st.checkbox("Add positions", value=True) + add_volume = st.checkbox("Add volume", value=True) + add_pnl = st.checkbox("Add PnL", value=True) + + run_backtesting_button = st.button("Run Backtesting!") + if run_backtesting_button: + config = strategy["config"](**st.session_state["strategy_params"]) + strategy = strategy["class"](config=config) + # TODO: add form for order amount, leverage, tp, sl, etc. + + market_data, positions = strategy.run_backtesting( + start='2021-04-01', + order_amount=50, + leverage=20, + initial_portfolio=100, + take_profit_multiplier=2.3, + stop_loss_multiplier=1.2, + time_limit=60 * 60 * 3, + std_span=None, + ) + strategy_analysis = StrategyAnalysis( + positions=positions, + candles_df=market_data, + ) + st.text(strategy_analysis.text_report()) + # TODO: check why the pnl is not being plotted + strategy_analysis.create_base_figure(volume=add_volume, positions=add_positions, trade_pnl=add_pnl) + st.plotly_chart(strategy_analysis.figure(), use_container_width=True) with optimize: - pass + # TODO: + # * Add videos explaining how to use the optimization tool, quick intro to optuna, etc in a toggle + with st.container(): + c1, c2 = st.columns([1, 1]) + with c1: + strategies = load_directional_strategies(constants.DIRECTIONAL_STRATEGIES_PATH) + strategy_to_optimize = st.selectbox("Select strategy to optimize", strategies.keys()) + with c2: + today = datetime.datetime.today() + # TODO: add hints about the study name + STUDY_NAME = st.text_input("Study name", + f"{strategy_to_optimize}_study_{today.day:02d}-{today.month:02d}-{today.year}") + c1, c2 = st.columns([4, 1]) + with c1: + # TODO: every time that we save and run the optimizations, we should save the code in a file + # so the user then can correlate the results with the code + st.session_state.strategy_code = st_ace(key="create_code", + value=strategy_optimization_template( + strategy_info=strategies[strategy_to_optimize]), + language='python', + keybinding='vscode', + theme='pastel_on_dark') + with c2: + if st.button("Save optimizations"): + save_file(name=f"{STUDY_NAME}.py", content=st.session_state.strategy_code, + path=constants.OPTIMIZATIONS_PATH) + if st.button("Run optimizations"): + study = optuna.create_study(direction="maximize", study_name=STUDY_NAME, + storage="sqlite:///data/backtesting/backtesting_report.db", + load_if_exists=True) + objective = get_function_from_file(file_path=f"{constants.OPTIMIZATIONS_PATH}/{STUDY_NAME}.py", + function_name="objective") + + + def optimization_process(): + study.optimize(objective, n_trials=2000) + + + optimization_thread = threading.Thread(target=optimization_process) + optimization_thread.start() + if st.button("Launch optuna dashboard"): + os_utils.execute_bash_command(f"optuna-dashboard sqlite:///data/backtesting/backtesting_report.db") + webbrowser.open("http://127.0.0.1:8080/dashboard", new=2) with analyze: + # TODO: + # * Add graphs for all backtesting results + # * Add management of backtesting results (delete, rename, save, share, upload s3, etc) pass From 181c962e5e583b43eaecf240b07e562ceecfc9f9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Jul 2023 11:52:05 +0200 Subject: [PATCH 59/64] (feat) allow multiple pages code gen --- pages/9_⚙️_Backtesting.py | 49 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py index c335b06..9d313c3 100644 --- a/pages/9_⚙️_Backtesting.py +++ b/pages/9_⚙️_Backtesting.py @@ -97,7 +97,7 @@ with optimize: # TODO: # * Add videos explaining how to use the optimization tool, quick intro to optuna, etc in a toggle with st.container(): - c1, c2 = st.columns([1, 1]) + c1, c2, c3 = st.columns([1, 1, 1]) with c1: strategies = load_directional_strategies(constants.DIRECTIONAL_STRATEGIES_PATH) strategy_to_optimize = st.selectbox("Select strategy to optimize", strategies.keys()) @@ -106,26 +106,27 @@ with optimize: # TODO: add hints about the study name STUDY_NAME = st.text_input("Study name", f"{strategy_to_optimize}_study_{today.day:02d}-{today.month:02d}-{today.year}") - c1, c2 = st.columns([4, 1]) - with c1: - # TODO: every time that we save and run the optimizations, we should save the code in a file - # so the user then can correlate the results with the code - st.session_state.strategy_code = st_ace(key="create_code", - value=strategy_optimization_template( - strategy_info=strategies[strategy_to_optimize]), - language='python', - keybinding='vscode', - theme='pastel_on_dark') - with c2: - if st.button("Save optimizations"): - save_file(name=f"{STUDY_NAME}.py", content=st.session_state.strategy_code, - path=constants.OPTIMIZATIONS_PATH) - if st.button("Run optimizations"): - study = optuna.create_study(direction="maximize", study_name=STUDY_NAME, - storage="sqlite:///data/backtesting/backtesting_report.db", - load_if_exists=True) - objective = get_function_from_file(file_path=f"{constants.OPTIMIZATIONS_PATH}/{STUDY_NAME}.py", - function_name="objective") + with c3: + generate_optimization_code_button = st.button("Generate Optimization Code") + + if generate_optimization_code_button: + c1, c2 = st.columns([4, 1]) + with c1: + # TODO: every time that we save and run the optimizations, we should save the code in a file + # so the user then can correlate the results with the code. + optimization_code = st_ace(key="create_optimization_code", + value=strategy_optimization_template(strategy_info=strategies[strategy_to_optimize]), + language='python', + keybinding='vscode', + theme='pastel_on_dark') + with c2: + if st.button("Run optimization"): + save_file(name=f"{STUDY_NAME}.py", content=optimization_code, path=constants.OPTIMIZATIONS_PATH) + study = optuna.create_study(direction="maximize", study_name=STUDY_NAME, + storage="sqlite:///data/backtesting/backtesting_report.db", + load_if_exists=True) + objective = get_function_from_file(file_path=f"{constants.OPTIMIZATIONS_PATH}/{STUDY_NAME}.py", + function_name="objective") def optimization_process(): @@ -134,9 +135,9 @@ with optimize: optimization_thread = threading.Thread(target=optimization_process) optimization_thread.start() - if st.button("Launch optuna dashboard"): - os_utils.execute_bash_command(f"optuna-dashboard sqlite:///data/backtesting/backtesting_report.db") - webbrowser.open("http://127.0.0.1:8080/dashboard", new=2) + if st.button("Launch optuna dashboard"): + os_utils.execute_bash_command(f"optuna-dashboard sqlite:///data/backtesting/backtesting_report.db") + webbrowser.open("http://127.0.0.1:8080/dashboard", new=2) with analyze: # TODO: From c36df3f7e0b30de7653e3f22e0a6e5e8a81117b9 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Jul 2023 11:52:15 +0200 Subject: [PATCH 60/64] (feat) minor updates to format --- utils/file_templates.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/file_templates.py b/utils/file_templates.py index df0a062..8815006 100644 --- a/utils/file_templates.py +++ b/utils/file_templates.py @@ -66,7 +66,7 @@ def get_optuna_suggest_str(field_name: str, properties: Dict): "string": "trial.suggest_categorical", } config_num = f"('{field_name}', {properties.get('minimum', '_')}, {properties.get('maximum', '_')})" - config_cat = f"('{field_name}', ['{properties.get('default', '_')}', '_'])" + config_cat = f"('{field_name}', ['{properties.get('default', '_')}',])" optuna_trial_str = map_by_type[properties["type"]] + config_num if properties["type"] != "string" \ else map_by_type[properties["type"]] + config_cat @@ -80,8 +80,7 @@ def strategy_optimization_template(strategy_info: dict): field_schema = strategy_config.schema()["properties"] fields_str = [get_optuna_suggest_str(field_name, properties) for field_name, properties in field_schema.items()] fields_str = "".join([f" {field_str},\n" for field_str in fields_str]) - return f""" -import traceback + return f"""import traceback from optuna import TrialPruned From 67ad71cb548ef692bd6b1edf6094abb41c6384ae Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Jul 2023 13:06:57 +0200 Subject: [PATCH 61/64] (feat) refactor directional strategy template --- utils/file_templates.py | 54 +++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/utils/file_templates.py b/utils/file_templates.py index 8815006..ff8be84 100644 --- a/utils/file_templates.py +++ b/utils/file_templates.py @@ -1,60 +1,51 @@ from typing import Dict -def directional_strategy_template(strategy_name: str, - exchange: str, - trading_pair: str, - interval: str) -> str: +def directional_strategy_template(strategy_cls_name: str) -> str: + strategy_config_cls_name = f"{strategy_cls_name}Config" + sma_config_text = "{self.config.sma_length}" return f"""import pandas_ta as ta -import pandas as pd -import numpy as np +from pydantic import BaseModel, Field from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase +class {strategy_config_cls_name}(BaseModel): + exchange: str = Field(default="binance_perpetual") + trading_pair: str = Field(default="ETH-USDT") + interval: str = Field(default="1h") + sma_length: int = Field(default=20, ge=10, le=200) + # ... Add more fields here -class {strategy_name}(DirectionalStrategyBase): - # Define the attributes of the strategy - def __init__(self, - exchange="{exchange}", - trading_pair="{trading_pair}", - interval="{interval}"): - self.exchange = exchange - self.trading_pair = trading_pair - self.interval = interval + +class {strategy_cls_name}(DirectionalStrategyBase[{strategy_config_cls_name}]): def get_raw_data(self): # The method get candles will search for the data in the folder data/candles # If the data is not there, you can use the candles downloader to get the data df = self.get_candles( - exchange=self.exchange, - trading_pair=self.trading_pair, - interval=self.interval, + exchange=self.config.exchange, + trading_pair=self.config.trading_pair, + interval=self.config.interval, ) return df - def add_indicators(self, df): - df.ta.sma(length=20, append=True) + def preprocessing(self, df): + df.ta.sma(length=self.config.sma_length, append=True) # ... Add more indicators here # ... Check https://github.com/twopirllc/pandas-ta#indicators-by-category for more indicators # ... Use help(ta.indicator_name) to get more info return df - def add_signals(self, df): - # ... Do your own logic - random_series = pd.Series(np.random.randint(low=0, high=101, size=100)) - random_series_2 = pd.Series(np.random.randint(low=0, high=101, size=100)) - random_thold = np.random.randint(low=45, high=65) - random_thold_2 = np.random.randint(low=45, high=65) - + def predict(self, df): # Generate long and short conditions - macd_long_cond = (random_series > random_thold) & (random_series_2 > random_thold_2) - macd_short_cond = (random_series < random_thold) & (random_series_2 > random_thold_2) + long_cond = (df['close'] > df[f'SMA_{sma_config_text}']) + short_cond = (df['close'] < df[f'SMA_{sma_config_text}']) # Choose side df['side'] = 0 - df.loc[macd_long_cond, 'side'] = 1 - df.loc[macd_short_cond, 'side'] = -1 + df.loc[long_cond, 'side'] = 1 + df.loc[short_cond, 'side'] = -1 return df """ @@ -122,7 +113,6 @@ def objective(trial): trial.set_user_attr("avg_trading_time_in_hours", strategy_analysis.avg_trading_time_in_minutes() / 60) return strategy_analysis.net_profit_pct() except Exception as e: - # TODO: Log error traceback.print_exc() raise TrialPruned() """ From 1e9edb1d79e752f9ef3c18d82f4d36d9f348f68e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Jul 2023 13:07:10 +0200 Subject: [PATCH 62/64] (feat) add todos to directional strategy base --- quants_lab/strategy/directional_strategy_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/quants_lab/strategy/directional_strategy_base.py b/quants_lab/strategy/directional_strategy_base.py index cb623d8..224e5dc 100644 --- a/quants_lab/strategy/directional_strategy_base.py +++ b/quants_lab/strategy/directional_strategy_base.py @@ -10,6 +10,9 @@ ConfigType = TypeVar("ConfigType", bound=BaseModel) class DirectionalStrategyBase(Generic[ConfigType]): + # TODO: + # * Add a data structure to request candles from CSV files as part of the config + # * Evaluate to move the get data outside the backtesting to optimize the performance. def __init__(self, config: ConfigType): self.config = config @@ -62,7 +65,6 @@ class DirectionalStrategyBase(Generic[ConfigType]): std_span, order_amount=100, leverage=20, initial_portfolio=1000, taker_fee=0.0003, maker_fee=0.00012, start: Optional[str] = None, end: Optional[str] = None): - # TODO: Evaluate to move the get data outside the backtesting to optimize the performance. df = self.get_data(start=start, end=end) df = self.preprocessing(df) df = self.predict(df) From f9a2e740de757c219b6bdee97a60c9e5a7d87a1d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Jul 2023 13:07:20 +0200 Subject: [PATCH 63/64] (feat) implement create directional strategy --- pages/9_⚙️_Backtesting.py | 54 ++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/pages/9_⚙️_Backtesting.py b/pages/9_⚙️_Backtesting.py index 9d313c3..e768cf7 100644 --- a/pages/9_⚙️_Backtesting.py +++ b/pages/9_⚙️_Backtesting.py @@ -8,7 +8,7 @@ from streamlit_ace import st_ace import constants from quants_lab.strategy.strategy_analysis import StrategyAnalysis from utils import os_utils -from utils.file_templates import strategy_optimization_template +from utils.file_templates import strategy_optimization_template, directional_strategy_template from utils.os_utils import load_directional_strategies, save_file, get_function_from_file import optuna @@ -25,7 +25,32 @@ st.title("⚙️ Backtesting") create, modify, backtest, optimize, analyze = st.tabs(["Create", "Modify", "Backtest", "Optimize", "Analyze"]) with create: - pass + # TODO: + # * Add videos explaining how to the triple barrier method works and how the backtesting is designed, + # link to video of how to create a strategy, etc in a toggle. + # * Add functionality to start strategy creation from scratch or by duplicating an existing one + c1, c2 = st.columns([4, 1]) + with c1: + # TODO: Allow change the strategy name and see the effect in the code + strategy_name = st.text_input("Strategy class name", value="CustomStrategy") + with c2: + update_strategy_name = st.button("Update Strategy Name") + + c1, c2 = st.columns([4, 1]) + with c1: + # TODO: every time that we save and run the optimizations, we should save the code in a file + # so the user then can correlate the results with the code. + if update_strategy_name: + st.session_state.directional_strategy_code = st_ace(key="create_directional_strategy", + value=directional_strategy_template(strategy_name), + language='python', + keybinding='vscode', + theme='pastel_on_dark') + with c2: + if st.button("Save strategy"): + save_file(name=f"{strategy_name.lower()}.py", content=st.session_state.directional_strategy_code, + path=constants.DIRECTIONAL_STRATEGIES_PATH) + st.success(f"Strategy {strategy_name} saved successfully") with modify: pass @@ -108,20 +133,25 @@ with optimize: f"{strategy_to_optimize}_study_{today.day:02d}-{today.month:02d}-{today.year}") with c3: generate_optimization_code_button = st.button("Generate Optimization Code") + if st.button("Launch optuna dashboard"): + os_utils.execute_bash_command(f"optuna-dashboard sqlite:///data/backtesting/backtesting_report.db") + webbrowser.open("http://127.0.0.1:8080/dashboard", new=2) + c1, c2 = st.columns([4, 1]) if generate_optimization_code_button: - c1, c2 = st.columns([4, 1]) with c1: # TODO: every time that we save and run the optimizations, we should save the code in a file # so the user then can correlate the results with the code. - optimization_code = st_ace(key="create_optimization_code", - value=strategy_optimization_template(strategy_info=strategies[strategy_to_optimize]), + st.session_state.optimization_code = st_ace(key="create_optimization_code", + value=strategy_optimization_template( + strategy_info=strategies[strategy_to_optimize]), language='python', keybinding='vscode', theme='pastel_on_dark') + if "optimization_code" in st.session_state: with c2: if st.button("Run optimization"): - save_file(name=f"{STUDY_NAME}.py", content=optimization_code, path=constants.OPTIMIZATIONS_PATH) + save_file(name=f"{STUDY_NAME}.py", content=st.session_state.optimization_code, path=constants.OPTIMIZATIONS_PATH) study = optuna.create_study(direction="maximize", study_name=STUDY_NAME, storage="sqlite:///data/backtesting/backtesting_report.db", load_if_exists=True) @@ -129,15 +159,13 @@ with optimize: function_name="objective") - def optimization_process(): - study.optimize(objective, n_trials=2000) + def optimization_process(): + study.optimize(objective, n_trials=2000) - optimization_thread = threading.Thread(target=optimization_process) - optimization_thread.start() - if st.button("Launch optuna dashboard"): - os_utils.execute_bash_command(f"optuna-dashboard sqlite:///data/backtesting/backtesting_report.db") - webbrowser.open("http://127.0.0.1:8080/dashboard", new=2) + optimization_thread = threading.Thread(target=optimization_process) + optimization_thread.start() + with analyze: # TODO: From 3905ed25a1f40c62c2a4f54f7c885caf4ab34fd5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Jul 2023 13:22:33 +0200 Subject: [PATCH 64/64] (feat) remove docker manager since we are using the new library --- utils/docker_manager.py | 91 ----------------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 utils/docker_manager.py diff --git a/utils/docker_manager.py b/utils/docker_manager.py deleted file mode 100644 index cd355c5..0000000 --- a/utils/docker_manager.py +++ /dev/null @@ -1,91 +0,0 @@ -import subprocess -from typing import Dict -import yaml - -import constants -from utils import os_utils - - -class DockerManager: - def __init__(self): - pass - - @staticmethod - def get_active_containers(): - cmd = "docker ps --format '{{.Names}}'" - output = subprocess.check_output(cmd, shell=True) - backtestings = [container for container in output.decode().split()] - return backtestings - - @staticmethod - def get_exited_containers(): - cmd = "docker ps --filter status=exited --format '{{.Names}}'" - output = subprocess.check_output(cmd, shell=True) - containers = output.decode().split() - return containers - - @staticmethod - def clean_exited_containers(): - cmd = "docker container prune --force" - subprocess.Popen(cmd, shell=True) - - def stop_active_containers(self): - containers = self.get_active_containers() - for container in containers: - cmd = f"docker stop {container}" - subprocess.Popen(cmd, shell=True) - - def stop_container(self, container_name): - cmd = f"docker stop {container_name}" - subprocess.Popen(cmd, shell=True) - - def start_container(self, container_name): - cmd = f"docker start {container_name}" - subprocess.Popen(cmd, shell=True) - - def remove_container(self, container_name): - cmd = f"docker rm {container_name}" - subprocess.Popen(cmd, shell=True) - - def create_download_candles_container(self, candles_config: Dict): - os_utils.dump_dict_to_yaml(candles_config, constants.DOWNLOAD_CANDLES_CONFIG_YML) - command = ["docker", "compose", "-p", "data_downloader", "-f", - "hummingbot_files/compose_files/data-downloader-compose.yml", "up", "-d"] - subprocess.Popen(command) - - def create_broker(self): - command = ["docker", "compose", "-p", "hummingbot-broker", "-f", - "hummingbot_files/compose_files/broker-compose.yml", "up", "-d", "--remove-orphans"] - subprocess.Popen(command) - - def create_hummingbot_instance(self, instance_name): - bot_name = f"hummingbot-{instance_name}" - base_conf_folder = f"{constants.BOTS_FOLDER}/data_downloader/conf" - bot_folder = f"{constants.BOTS_FOLDER}/{bot_name}" - if not os_utils.directory_exists(bot_folder): - create_folder_command = ["mkdir", "-p", bot_folder] - create_folder_task = subprocess.Popen(create_folder_command) - create_folder_task.wait() - command = ["cp", "-rf", base_conf_folder, bot_folder] - copy_folder_task = subprocess.Popen(command) - copy_folder_task.wait() - conf_file_path = f"{bot_folder}/conf/conf_client.yml" - config = os_utils.read_yaml_file(conf_file_path) - config['instance_id'] = bot_name - os_utils.dump_dict_to_yaml(config, conf_file_path) - # TODO: Mount script folder for custom scripts - create_container_command = ["docker", "run", "-it", "-d", "--log-opt", "max-size=10m", "--log-opt", - "max-file=5", - "--name", bot_name, - "--network", "host", - "-v", f"./{bot_folder}/conf:/home/hummingbot/conf", - "-v", f"./{bot_folder}/conf/connectors:/home/hummingbot/conf/connectors", - "-v", f"./{bot_folder}/conf/strategies:/home/hummingbot/conf/strategies", - "-v", f"./{bot_folder}/logs:/home/hummingbot/logs", - "-v", "./data/:/home/hummingbot/data", - # "-v", f"./{bot_folder}/scripts:/home/hummingbot/scripts", - "-v", f"./{bot_folder}/certs:/home/hummingbot/certs", - "-e", "CONFIG_PASSWORD=a", - "dardonacci/hummingbot:development"] - - subprocess.Popen(create_container_command)