diff --git a/.gitignore b/.gitignore index 5065581..f6b5a89 100644 --- a/.gitignore +++ b/.gitignore @@ -134,7 +134,7 @@ dmypy.json # Optimize studies and strategies quants_lab/optimizations/* -quants_lab/strategy/experiments/* +quants_lab/controllers/* # Master bot template user-added configs hummingbot_files/templates/master_bot_conf/conf/* diff --git a/constants.py b/constants.py index bdb8830..d56292d 100644 --- a/constants.py +++ b/constants.py @@ -1,6 +1,7 @@ CANDLES_DATA_PATH = "data/candles" DOWNLOAD_CANDLES_CONFIG_YML = "hummingbot_files/scripts_configs/data_downloader_config.yml" BOTS_FOLDER = "hummingbot_files/bot_configs" -DIRECTIONAL_STRATEGIES_PATH = "quants_lab/strategy/experiments" +CONTROLLERS_PATH = "quants_lab/controllers" +CONTROLLERS_CONFIG_PATH = "hummingbot_files/controller_configs" OPTIMIZATIONS_PATH = "quants_lab/optimizations" HUMMINGBOT_TEMPLATES = "hummingbot_files/templates" diff --git a/environment_conda.yml b/environment_conda.yml index 2662b5c..d35aeb4 100644 --- a/environment_conda.yml +++ b/environment_conda.yml @@ -1,11 +1,14 @@ name: dashboard channels: + - defaults - conda-forge dependencies: - - python=3.9 + - python=3.10 - sqlalchemy + - pydantic=1.9.* - pip - pip: + - hummingbot - streamlit - watchdog - plotly @@ -13,7 +16,7 @@ dependencies: - glom - defillama - statsmodels - - pandas_ta + - pandas_ta==0.3.14b - pyyaml - commlib-py - jupyter diff --git a/hummingbot_files/controller_configs/.gitignore b/hummingbot_files/controller_configs/.gitignore new file mode 100644 index 0000000..42780ec --- /dev/null +++ b/hummingbot_files/controller_configs/.gitignore @@ -0,0 +1 @@ + * \ No newline at end of file diff --git a/hummingbot_files/templates/master_bot_conf/scripts/strategy_v2_launcher.py b/hummingbot_files/templates/master_bot_conf/scripts/strategy_v2_launcher.py new file mode 100644 index 0000000..2f67432 --- /dev/null +++ b/hummingbot_files/templates/master_bot_conf/scripts/strategy_v2_launcher.py @@ -0,0 +1,81 @@ +import inspect +import os +import importlib.util + +from hummingbot.core.data_type.common import OrderType, PositionMode, TradeType +from hummingbot.smart_components.strategy_frameworks.data_types import ( + ExecutorHandlerStatus, +) +from hummingbot.smart_components.strategy_frameworks.directional_trading import DirectionalTradingControllerBase, \ + DirectionalTradingControllerConfigBase, DirectionalTradingExecutorHandler +from hummingbot.smart_components.utils import ConfigEncoderDecoder +from hummingbot.strategy.script_strategy_base import ScriptStrategyBase + + +def load_controllers(path): + controllers = {} + for filename in os.listdir(path): + if filename.endswith('.py') and "__init__" not in filename: + module_name = filename[:-3] # strip the .py to get the module name + controllers[module_name] = {"module": module_name} + file_path = os.path.join(path, filename) + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + for name, cls in inspect.getmembers(module, inspect.isclass): + if issubclass(cls, DirectionalTradingControllerBase) and cls is not DirectionalTradingControllerBase: + controllers[module_name]["class"] = cls + if issubclass(cls, DirectionalTradingControllerConfigBase) and cls is not DirectionalTradingControllerConfigBase: + controllers[module_name]["config"] = cls + return controllers + + +def initialize_controller_from_config(encoder_decoder: ConfigEncoderDecoder, + all_controllers_info: dict, + controller_config_file: str): + config = encoder_decoder.yaml_load(f"conf/controllers_config/{controller_config_file}") + controller_info = all_controllers_info[config["strategy_name"]] + config_instance = controller_info["config"](**config) + controller_class = controller_info["class"](config_instance) + return controller_class + + +class StrategyV2Launcher(ScriptStrategyBase): + controller_configs = os.getenv("controller_configs", "bollinger_8044.yml,bollinger_8546.yml,bollinger_8883.yml") + controllers = {} + markets = {} + executor_handlers = {} + encoder_decoder = ConfigEncoderDecoder(TradeType, PositionMode, OrderType) + controllers_info = load_controllers("hummingbot/smart_components/controllers") + + for controller_config in controller_configs.split(","): + controller = initialize_controller_from_config(encoder_decoder, controllers_info, controller_config) + markets = controller.update_strategy_markets_dict(markets) + controllers[controller_config] = controller + + def __init__(self, connectors): + super().__init__(connectors) + for controller_config, controller in self.controllers.items(): + self.executor_handlers[controller_config] = DirectionalTradingExecutorHandler(strategy=self, controller=controller) + + def on_stop(self): + for executor_handler in self.executor_handlers.values(): + executor_handler.stop() + + def on_tick(self): + """ + This shows you how you can start meta controllers. You can run more than one at the same time and based on the + market conditions, you can orchestrate from this script when to stop or start them. + """ + for executor_handler in self.executor_handlers.values(): + if executor_handler.status == ExecutorHandlerStatus.NOT_STARTED: + executor_handler.start() + + def format_status(self) -> str: + if not self.ready_to_trade: + return "Market connectors are not ready." + lines = [] + for controller_config, executor_handler in self.executor_handlers.items(): + lines.extend([f"Strategy: {executor_handler.controller.config.strategy_name} | Config: {controller_config}", + executor_handler.to_format_status()]) + return "\n".join(lines) diff --git a/main.py b/main.py index bf8b140..3895c7e 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,7 @@ show_pages( Page("main.py", "Hummingbot Dashboard", "πŸ“Š"), Section("Bot Orchestration", "πŸ™"), Page("pages/master_conf/app.py", "Credentials", "πŸ—οΈ"), + Page("pages/launch_bot/app.py", "Launch Bot", "πŸ™Œ"), Page("pages/bot_orchestration/app.py", "Instances", "πŸ¦…"), Page("pages/file_manager/app.py", "Strategy Configs", "πŸ—‚"), Section("Backtest Manager", "βš™οΈ"), @@ -17,6 +18,7 @@ show_pages( Page("pages/backtest_manager/create.py", "Create", "βš”οΈ"), Page("pages/backtest_manager/optimize.py", "Optimize", "πŸ§ͺ"), Page("pages/backtest_manager/analyze.py", "Analyze", "πŸ”¬"), + Page("pages/backtest_manager/analyze_v2.py", "Analyze v2", "πŸ”¬"), Page("pages/backtest_manager/simulate.py", "Simulate", "πŸ“ˆ"), Section("Community Pages", "πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦"), Page("pages/strategy_performance/app.py", "Strategy Performance", "πŸš€"), diff --git a/pages/backtest_manager/analyze.py b/pages/backtest_manager/analyze.py index c7b80e0..33b0b91 100644 --- a/pages/backtest_manager/analyze.py +++ b/pages/backtest_manager/analyze.py @@ -1,3 +1,9 @@ +from hummingbot.core.data_type.common import PositionMode, TradeType, OrderType +from hummingbot.data_feed.candles_feed.candles_factory import CandlesConfig +from hummingbot.smart_components.strategy_frameworks.data_types import OrderLevel +from hummingbot.smart_components.strategy_frameworks.directional_trading import DirectionalTradingBacktestingEngine +from hummingbot.smart_components.utils import ConfigEncoderDecoder + import constants import os import json @@ -6,7 +12,7 @@ import streamlit as st from quants_lab.strategy.strategy_analysis import StrategyAnalysis from utils.graphs import BacktestingGraphs from utils.optuna_database_manager import OptunaDBManager -from utils.os_utils import load_directional_strategies +from utils.os_utils import load_controllers, dump_dict_to_yaml from utils.st_utils import initialize_st_page initialize_st_page(title="Analyze", icon="πŸ”¬", initial_sidebar_state="collapsed") @@ -53,32 +59,36 @@ else: trial_selected = st.selectbox("Select a trial to backtest", list(trials.keys())) trial = trials[trial_selected] # Transform trial config in a dictionary - trial_config = json.loads(trial["config"]) + encoder_decoder = ConfigEncoderDecoder(TradeType, OrderType, PositionMode) + trial_config = encoder_decoder.decode(json.loads(trial["config"])) # Strategy parameters section st.write("## Strategy parameters") # Load strategies (class, config, module) - strategies = load_directional_strategies(constants.DIRECTIONAL_STRATEGIES_PATH) + controllers = load_controllers(constants.CONTROLLERS_PATH) # Select strategy - strategy = strategies[trial_config["name"]] + controller = controllers[trial_config["strategy_name"]] # Get field schema - field_schema = strategy["config"].schema()["properties"] + field_schema = controller["config"].schema()["properties"] + c1, c2 = st.columns([5, 1]) # Render every field according to schema with c1: columns = st.columns(4) column_index = 0 for field_name, properties in field_schema.items(): - field_type = properties["type"] + field_type = properties.get("type", "string") field_value = trial_config[field_name] with columns[column_index]: - if field_type in ["number", "integer"]: + if field_type == "array" or field_name == "position_mode": + pass + elif field_type in ["number", "integer"]: field_value = st.number_input(field_name, value=field_value, min_value=properties.get("minimum"), - max_value=properties.get("maximum"), + # max_value=properties.get("maximum"), key=field_name) - elif field_type == "string": + elif field_type in ["string"]: field_value = st.text_input(field_name, value=field_value) elif field_type == "boolean": # TODO: Add support for boolean fields in optimize tab @@ -86,6 +96,13 @@ else: else: raise ValueError(f"Field type {field_type} not supported") try: + # TODO: figure out how to make this configurable + if field_name == "candles_config": + candles_config = [CandlesConfig(**value) for value in field_value] + st.session_state["strategy_params"][field_name] = candles_config + elif field_name == "order_levels": + order_levels = [OrderLevel(**value) for value in field_value] + st.session_state["strategy_params"][field_name] = order_levels st.session_state["strategy_params"][field_name] = field_value except KeyError as e: pass @@ -97,60 +114,53 @@ else: # Backtesting parameters section st.write("## Backtesting parameters") - # Get every trial params - # TODO: Filter only from selected study + # # Get every trial params + # # TODO: Filter only from selected study backtesting_configs = opt_db.load_params() - # Get trial backtesting params + # # Get trial backtesting params backtesting_params = backtesting_configs[trial_selected] - col1, col2, col3 = st.columns(3) + col1, col2, col3, col4 = st.columns([1, 1, 1, 0.5]) with col1: - selected_order_amount = st.number_input("Order amount", - value=50.0, - min_value=0.1, - max_value=999999999.99) - selected_leverage = st.number_input("Leverage", - value=10, - min_value=1, - max_value=200) + trade_cost = st.number_input("Trade cost", + value=0.0006, + min_value=0.0001, format="%.4f",) with col2: - selected_initial_portfolio = st.number_input("Initial portfolio", - value=10000.00, - min_value=1.00, - max_value=999999999.99) - selected_time_limit = st.number_input("Time Limit", - value=60 * 60 * backtesting_params["time_limit"]["param_value"], - min_value=60 * 60 * float(backtesting_params["time_limit"]["low"]), - max_value=60 * 60 * float(backtesting_params["time_limit"]["high"])) + initial_portfolio_usd = st.number_input("Initial portfolio usd", + value=10000.00, + min_value=1.00, + max_value=999999999.99) with col3: - selected_tp_multiplier = st.number_input("Take Profit Multiplier", - value=backtesting_params["take_profit_multiplier"]["param_value"], - min_value=backtesting_params["take_profit_multiplier"]["low"], - max_value=backtesting_params["take_profit_multiplier"]["high"]) - selected_sl_multiplier = st.number_input("Stop Loss Multiplier", - value=backtesting_params["stop_loss_multiplier"]["param_value"], - min_value=backtesting_params["stop_loss_multiplier"]["low"], - max_value=backtesting_params["stop_loss_multiplier"]["high"]) - - if st.button("Run Backtesting!"): - config = strategy["config"](**st.session_state["strategy_params"]) - strategy = strategy["class"](config=config) + start = st.text_input("Start", value="2023-01-01") + end = st.text_input("End", value="2023-08-01") + c1, c2 = st.columns([1, 1]) + with col4: + deploy_button = st.button("πŸ’ΎSave controller config!") + config = controller["config"](**st.session_state["strategy_params"]) + controller = controller["class"](config=config) + if deploy_button: + encoder_decoder.yaml_dump(config.dict(), + f"hummingbot_files/controller_configs/{config.strategy_name}_{trial_selected}.yml") + # DockerManager().create_hummingbot_instance(instance_name=config.strategy_name, + # base_conf_folder=f"{constants.HUMMINGBOT_TEMPLATES}/master_bot_conf/.", + # target_conf_folder=f"{constants.BOTS_FOLDER}/{config.strategy_name}/.", + # controllers_folder="quants_lab/controllers", + # controllers_config_folder="hummingbot_files/controller_configs", + # image="dardonacci/hummingbot") + run_backtesting_button = st.button("βš™οΈRun Backtesting!") + if run_backtesting_button: try: - market_data, positions = strategy.run_backtesting( - order_amount=selected_order_amount, - leverage=selected_order_amount, - initial_portfolio=selected_initial_portfolio, - take_profit_multiplier=selected_tp_multiplier, - stop_loss_multiplier=selected_sl_multiplier, - time_limit=selected_time_limit, - std_span=None, - ) + engine = DirectionalTradingBacktestingEngine(controller=controller) + engine.load_controller_data("./data/candles") + backtesting_results = engine.run_backtesting(initial_portfolio_usd=initial_portfolio_usd, + trade_cost=trade_cost, + start=start, end=end) strategy_analysis = StrategyAnalysis( - positions=positions, - candles_df=market_data, + positions=backtesting_results["executors_df"], + candles_df=backtesting_results["processed_data"], ) metrics_container = bt_graphs.get_trial_metrics(strategy_analysis, add_positions=add_positions, - add_volume=add_volume, - add_pnl=add_pnl) + add_volume=add_volume) + except FileNotFoundError: st.warning(f"The requested candles could not be found.") diff --git a/pages/backtest_manager/analyze_v2.py b/pages/backtest_manager/analyze_v2.py new file mode 100644 index 0000000..e5111ce --- /dev/null +++ b/pages/backtest_manager/analyze_v2.py @@ -0,0 +1,227 @@ +from hummingbot.core.data_type.common import PositionMode, TradeType, OrderType +from hummingbot.data_feed.candles_feed.candles_factory import CandlesConfig +from hummingbot.smart_components.strategy_frameworks.data_types import OrderLevel, TripleBarrierConf +from hummingbot.smart_components.strategy_frameworks.directional_trading import DirectionalTradingBacktestingEngine +from hummingbot.smart_components.utils import ConfigEncoderDecoder + +import constants +import os +import json +import streamlit as st +from decimal import Decimal + +from quants_lab.strategy.strategy_analysis import StrategyAnalysis +from utils.graphs import BacktestingGraphs +from utils.optuna_database_manager import OptunaDBManager +from utils.os_utils import load_controllers +from utils.st_utils import initialize_st_page + + +initialize_st_page(title="Analyze", icon="πŸ”¬", initial_sidebar_state="collapsed") + + +@st.cache_resource +def get_databases(): + sqlite_files = [db_name for db_name in os.listdir("data/backtesting") if db_name.endswith(".db")] + databases_list = [OptunaDBManager(db) for db in sqlite_files] + databases_dict = {database.db_name: database for database in databases_list} + return [x.db_name for x in databases_dict.values() if x.status == 'OK'] + + +def initialize_session_state_vars(): + if "strategy_params" not in st.session_state: + st.session_state.strategy_params = {} + if "backtesting_params" not in st.session_state: + st.session_state.backtesting_params = {} + + +initialize_session_state_vars() +dbs = get_databases() +if not dbs: + st.warning("We couldn't find any Optuna database.") + selected_db_name = None + selected_db = None +else: + # Select database from selectbox + selected_db = st.selectbox("Select your database:", dbs) + # Instantiate database manager + opt_db = OptunaDBManager(selected_db) + # Load studies + studies = opt_db.load_studies() + # Choose study + study_selected = st.selectbox("Select a study:", studies.keys()) + # Filter trials from selected study + merged_df = opt_db.merged_df[opt_db.merged_df["study_name"] == study_selected] + filters_column, scatter_column = st.columns([1, 6]) + with filters_column: + accuracy = st.slider("Accuracy", min_value=0.0, max_value=1.0, value=[0.4, 1.0], step=0.01) + net_profit = st.slider("Net PNL (%)", min_value=merged_df["net_pnl_pct"].min(), max_value=merged_df["net_pnl_pct"].max(), + value=[merged_df["net_pnl_pct"].min(), merged_df["net_pnl_pct"].max()], step=0.01) + max_drawdown = st.slider("Max Drawdown (%)", min_value=merged_df["max_drawdown_pct"].min(), max_value=merged_df["max_drawdown_pct"].max(), + value=[merged_df["max_drawdown_pct"].min(), merged_df["max_drawdown_pct"].max()], step=0.01) + total_positions = st.slider("Total Positions", min_value=merged_df["total_positions"].min(), max_value=merged_df["total_positions"].max(), + value=[merged_df["total_positions"].min(), merged_df["total_positions"].max()], step=1) + net_profit_filter = (merged_df["net_pnl_pct"] >= net_profit[0]) & (merged_df["net_pnl_pct"] <= net_profit[1]) + accuracy_filter = (merged_df["accuracy"] >= accuracy[0]) & (merged_df["accuracy"] <= accuracy[1]) + max_drawdown_filter = (merged_df["max_drawdown_pct"] >= max_drawdown[0]) & (merged_df["max_drawdown_pct"] <= max_drawdown[1]) + total_positions_filter = (merged_df["total_positions"] >= total_positions[0]) & (merged_df["total_positions"] <= total_positions[1]) + with scatter_column: + bt_graphs = BacktestingGraphs(merged_df[net_profit_filter & accuracy_filter & max_drawdown_filter & total_positions_filter]) + # Show and compare all of the study trials + st.plotly_chart(bt_graphs.pnl_vs_maxdrawdown(), use_container_width=True) + # Get study trials + trials = studies[study_selected] + # Choose trial + trial_selected = st.selectbox("Select a trial to backtest", list(trials.keys())) + trial = trials[trial_selected] + # Transform trial config in a dictionary + encoder_decoder = ConfigEncoderDecoder(TradeType, OrderType, PositionMode) + trial_config = encoder_decoder.decode(json.loads(trial["config"])) + + # Strategy parameters section + st.write("## Strategy parameters") + # Load strategies (class, config, module) + controllers = load_controllers(constants.CONTROLLERS_PATH) + # Select strategy + controller = controllers[trial_config["strategy_name"]] + # Get field schema + field_schema = controller["config"].schema()["properties"] + + columns = st.columns(4) + column_index = 0 + for field_name, properties in field_schema.items(): + field_type = properties.get("type", "string") + field_value = trial_config[field_name] + if field_name not in ["candles_config", "order_levels", "position_mode"]: + with columns[column_index]: + if field_type in ["number", "integer"]: + field_value = st.number_input(field_name, + value=field_value, + min_value=properties.get("minimum"), + max_value=properties.get("maximum"), + key=field_name) + elif field_type == "string": + field_value = st.text_input(field_name, value=field_value) + elif field_type == "boolean": + # TODO: Add support for boolean fields in optimize tab + field_value = st.checkbox(field_name, value=field_value) + else: + raise ValueError(f"Field type {field_type} not supported") + else: + if field_name == "candles_config": + st.write("---") + st.write(f"## Candles Config:") + candles = [] + for i, candles_config in enumerate(field_value): + st.write(f"#### Candle {i}:") + c11, c12, c13, c14 = st.columns(4) + with c11: + connector = st.text_input("Connector", value=candles_config["connector"]) + with c12: + trading_pair = st.text_input("Trading pair", value=candles_config["trading_pair"]) + with c13: + interval = st.text_input("Interval", value=candles_config["interval"]) + with c14: + max_records = st.number_input("Max records", value=candles_config["max_records"]) + st.write("---") + candles.append(CandlesConfig(connector=connector, trading_pair=trading_pair, interval=interval, + max_records=max_records)) + field_value = candles + elif field_name == "order_levels": + new_levels = [] + st.write(f"## Order Levels:") + for order_level in field_value: + st.write(f"### Level {order_level['level']} {order_level['side'].name}") + ol_c1, ol_c2 = st.columns([5, 1]) + with ol_c1: + st.write("#### Triple Barrier config:") + c21, c22, c23, c24, c25 = st.columns(5) + triple_barrier_conf_level = order_level["triple_barrier_conf"] + with c21: + take_profit = st.number_input("Take profit", value=float(triple_barrier_conf_level["take_profit"]), + key=f"{order_level['level']}_{order_level['side'].name}_tp") + with c22: + stop_loss = st.number_input("Stop Loss", value=float(triple_barrier_conf_level["stop_loss"]), + key=f"{order_level['level']}_{order_level['side'].name}_sl") + with c23: + time_limit = st.number_input("Time Limit", value=triple_barrier_conf_level["time_limit"], + key=f"{order_level['level']}_{order_level['side'].name}_tl") + with c24: + ts_ap = st.number_input("Trailing Stop Activation Price", value=float(triple_barrier_conf_level["trailing_stop_activation_price_delta"]), + key=f"{order_level['level']}_{order_level['side'].name}_tsap", format="%.4f") + with c25: + ts_td = st.number_input("Trailing Stop Trailing Delta", value=float(triple_barrier_conf_level["trailing_stop_trailing_delta"]), + key=f"{order_level['level']}_{order_level['side'].name}_tstd", format="%.4f") + with ol_c2: + st.write("#### Position config:") + c31, c32 = st.columns(2) + with c31: + order_amount = st.number_input("Order amount USD", value=float(order_level["order_amount_usd"]), + key=f"{order_level['level']}_{order_level['side'].name}_oa") + with c32: + cooldown_time = st.number_input("Cooldown time", value=order_level["cooldown_time"], + key=f"{order_level['level']}_{order_level['side'].name}_cd") + triple_barrier_conf = TripleBarrierConf(stop_loss=Decimal(stop_loss), take_profit=Decimal(take_profit), + time_limit=time_limit, + trailing_stop_activation_price_delta=Decimal(ts_ap), + trailing_stop_trailing_delta=Decimal(ts_td), + open_order_type=OrderType.MARKET) + new_levels.append(OrderLevel(level=order_level["level"], side=order_level["side"], + order_amount_usd=order_amount, cooldown_time=cooldown_time, + triple_barrier_conf=triple_barrier_conf)) + st.write("---") + + field_value = new_levels + elif field_name == "position_mode": + field_value = PositionMode.HEDGE + else: + field_value = None + st.session_state["strategy_params"][field_name] = field_value + + column_index = (column_index + 1) % 4 + + st.write("### Backtesting period") + col1, col2, col3, col4 = st.columns([1, 1, 1, 0.5]) + with col1: + trade_cost = st.number_input("Trade cost", + value=0.0006, + min_value=0.0001, format="%.4f", ) + with col2: + initial_portfolio_usd = st.number_input("Initial portfolio usd", + value=10000.00, + min_value=1.00, + max_value=999999999.99) + with col3: + start = st.text_input("Start", value="2023-01-01") + end = st.text_input("End", value="2023-08-01") + c1, c2 = st.columns([1, 1]) + with col4: + add_positions = st.checkbox("Add positions", value=True) + add_volume = st.checkbox("Add volume", value=True) + add_pnl = st.checkbox("Add PnL", value=True) + save_config = st.button("πŸ’ΎSave controller config!") + config = controller["config"](**st.session_state["strategy_params"]) + controller = controller["class"](config=config) + if save_config: + encoder_decoder = ConfigEncoderDecoder(TradeType, OrderType, PositionMode) + encoder_decoder.yaml_dump(config.dict(), + f"hummingbot_files/controller_configs/{config.strategy_name}_{trial_selected}.yml") + run_backtesting_button = st.button("βš™οΈRun Backtesting!") + if run_backtesting_button: + try: + engine = DirectionalTradingBacktestingEngine(controller=controller) + engine.load_controller_data("./data/candles") + backtesting_results = engine.run_backtesting(initial_portfolio_usd=initial_portfolio_usd, + trade_cost=trade_cost, + start=start, end=end) + strategy_analysis = StrategyAnalysis( + positions=backtesting_results["executors_df"], + candles_df=backtesting_results["processed_data"], + ) + metrics_container = BacktestingGraphs(backtesting_results["processed_data"]).get_trial_metrics( + strategy_analysis, + add_positions=add_positions, + add_volume=add_volume) + + except FileNotFoundError: + st.warning(f"The requested candles could not be found.") diff --git a/pages/backtest_manager/optimize.py b/pages/backtest_manager/optimize.py index 1c7f496..f601d75 100644 --- a/pages/backtest_manager/optimize.py +++ b/pages/backtest_manager/optimize.py @@ -5,23 +5,17 @@ from types import SimpleNamespace import streamlit as st from streamlit_elements import elements, mui -import constants -from quants_lab.strategy.strategy_analysis import StrategyAnalysis from ui_components.dashboard import Dashboard -from ui_components.directional_strategies_file_explorer import DirectionalStrategiesFileExplorer -from ui_components.directional_strategy_creation_card import DirectionalStrategyCreationCard from ui_components.editor import Editor from ui_components.optimization_creation_card import OptimizationCreationCard from ui_components.optimization_run_card import OptimizationRunCard from ui_components.optimizations_file_explorer import OptimizationsStrategiesFileExplorer from utils import os_utils -from utils.os_utils import load_directional_strategies from utils.st_utils import initialize_st_page initialize_st_page(title="Optimize", icon="πŸ§ͺ", initial_sidebar_state="collapsed") - def run_optuna_dashboard(): os_utils.execute_bash_command(f"optuna-dashboard sqlite:///data/backtesting/backtesting_report.db") time.sleep(5) diff --git a/pages/backtest_manager/simulate.py b/pages/backtest_manager/simulate.py index 260cbf5..6231211 100644 --- a/pages/backtest_manager/simulate.py +++ b/pages/backtest_manager/simulate.py @@ -1,90 +1,151 @@ import time import webbrowser from types import SimpleNamespace +from decimal import Decimal import streamlit as st -from streamlit_elements import elements, mui +from hummingbot.core.data_type.common import TradeType, OrderType, PositionMode +from hummingbot.data_feed.candles_feed.candles_factory import CandlesConfig +from hummingbot.smart_components.strategy_frameworks.data_types import OrderLevel, TripleBarrierConf +from hummingbot.smart_components.strategy_frameworks.directional_trading import DirectionalTradingBacktestingEngine +from hummingbot.smart_components.utils import ConfigEncoderDecoder import constants from quants_lab.strategy.strategy_analysis import StrategyAnalysis -from ui_components.dashboard import Dashboard -from ui_components.directional_strategies_file_explorer import DirectionalStrategiesFileExplorer -from ui_components.directional_strategy_creation_card import DirectionalStrategyCreationCard -from ui_components.editor import Editor -from ui_components.optimization_creation_card import OptimizationCreationCard -from ui_components.optimization_run_card import OptimizationRunCard -from ui_components.optimizations_file_explorer import OptimizationsStrategiesFileExplorer -from utils import os_utils -from utils.os_utils import load_directional_strategies +from utils.graphs import BacktestingGraphs +from utils.os_utils import load_controllers from utils.st_utils import initialize_st_page - initialize_st_page(title="Simulate", icon="πŸ“ˆ", initial_sidebar_state="collapsed") # Start content here if "strategy_params" not in st.session_state: st.session_state.strategy_params = {} - # TODO: # * Add videos explaining how to the triple barrier method works and how the backtesting is designed, # link to video of how to create a strategy, etc in a toggle. # * Add performance analysis graphs of the backtesting run -strategies = load_directional_strategies(constants.DIRECTIONAL_STRATEGIES_PATH) -strategy_to_optimize = st.selectbox("Select strategy to backtest", strategies.keys()) -strategy = strategies[strategy_to_optimize] -strategy_config = strategy["config"] -field_schema = strategy_config.schema()["properties"] +controllers = load_controllers(constants.CONTROLLERS_PATH) +controller_to_optimize = st.selectbox("Select strategy to backtest", controllers.keys()) +controller = controllers[controller_to_optimize] +field_schema = controller["config"].schema()["properties"] st.write("## Strategy parameters") -c1, c2 = st.columns([5, 1]) -with c1: - columns = st.columns(4) - column_index = 0 - for field_name, properties in field_schema.items(): - field_type = properties["type"] +st.write("---") +columns = st.columns(4) +column_index = 0 +for field_name, properties in field_schema.items(): + field_type = properties.get("type", "string") + if field_name not in ["candles_config", "order_levels", "position_mode"]: + field_value = properties.get("default", "") with columns[column_index]: if field_type in ["number", "integer"]: field_value = st.number_input(field_name, - value=properties["default"], + value=field_value, min_value=properties.get("minimum"), max_value=properties.get("maximum"), key=field_name) elif field_type == "string": - field_value = st.text_input(field_name, value=properties["default"]) + field_value = st.text_input(field_name, value=field_value) elif field_type == "boolean": # TODO: Add support for boolean fields in optimize tab - field_value = st.checkbox(field_name, value=properties["default"]) + field_value = st.checkbox(field_name, value=field_value) else: raise ValueError(f"Field type {field_type} not supported") - st.session_state["strategy_params"][field_name] = field_value - column_index = (column_index + 1) % 4 -with c2: + else: + if field_name == "candles_config": + st.write("### Candles Config:") + c11, c12, c13, c14 = st.columns(4) + with c11: + connector = st.text_input("Connector", value="binance_perpetual") + with c12: + trading_pair = st.text_input("Trading pair", value="BTC-USDT") + with c13: + interval = st.text_input("Interval", value="3m") + with c14: + max_records = st.number_input("Max records", value=100000) + field_value = [CandlesConfig(connector=connector, trading_pair=trading_pair, interval=interval, + max_records=max_records)] + elif field_name == "order_levels": + st.write("### Triple Barrier config:") + c21, c22, c23, c24, c25 = st.columns(5) + with c21: + take_profit = st.number_input("Take profit", value=0.02) + with c22: + stop_loss = st.number_input("Stop Loss", value=0.01) + with c23: + time_limit = st.number_input("Time Limit", value=60 * 60 * 2) + with c24: + ts_ap = st.number_input("Trailing Stop Activation Price", value=0.01) + with c25: + ts_td = st.number_input("Trailing Stop Trailing Delta", value=0.005) + + st.write("### Position config:") + c31, c32 = st.columns(2) + with c31: + order_amount = st.number_input("Order amount USD", value=50) + with c32: + cooldown_time = st.number_input("Cooldown time", value=15) + triple_barrier_conf = TripleBarrierConf(stop_loss=Decimal(stop_loss), take_profit=Decimal(take_profit), + time_limit=time_limit, + trailing_stop_activation_price_delta=Decimal(ts_ap), + trailing_stop_trailing_delta=Decimal(ts_td), + open_order_type=OrderType.MARKET) + field_value = [ + OrderLevel(level=0, side=TradeType.BUY, order_amount_usd=order_amount, cooldown_time=cooldown_time, + triple_barrier_conf=triple_barrier_conf), + OrderLevel(level=0, side=TradeType.SELL, order_amount_usd=order_amount, cooldown_time=cooldown_time, + triple_barrier_conf=triple_barrier_conf), + ] + elif field_name == "position_mode": + field_value = PositionMode.HEDGE + st.session_state["strategy_params"][field_name] = field_value + + column_index = (column_index + 1) % 4 + +st.write("### Backtesting period") +col1, col2, col3, col4 = st.columns([1, 1, 1, 0.5]) +with col1: + trade_cost = st.number_input("Trade cost", + value=0.0006, + min_value=0.0001, format="%.4f", ) +with col2: + initial_portfolio_usd = st.number_input("Initial portfolio usd", + value=10000.00, + min_value=1.00, + max_value=999999999.99) +with col3: + start = st.text_input("Start", value="2023-01-01") + end = st.text_input("End", value="2023-08-01") +c1, c2 = st.columns([1, 1]) +with col4: add_positions = st.checkbox("Add positions", value=True) add_volume = st.checkbox("Add volume", value=True) add_pnl = st.checkbox("Add PnL", value=True) - - run_backtesting_button = st.button("Run Backtesting!") + save_config = st.button("πŸ’ΎSave controller config!") + config = controller["config"](**st.session_state["strategy_params"]) + controller = controller["class"](config=config) + if save_config: + encoder_decoder = ConfigEncoderDecoder(TradeType, OrderType, PositionMode) + # TODO: make this configurable + encoder_decoder.yaml_dump(config.dict(), + f"hummingbot_files/controller_configs/{config.strategy_name}.yml") + run_backtesting_button = st.button("βš™οΈRun Backtesting!") if run_backtesting_button: - config = strategy["config"](**st.session_state["strategy_params"]) - strategy = strategy["class"](config=config) - # TODO: add form for order amount, leverage, tp, sl, etc. + try: + engine = DirectionalTradingBacktestingEngine(controller=controller) + engine.load_controller_data("./data/candles") + backtesting_results = engine.run_backtesting(initial_portfolio_usd=initial_portfolio_usd, + trade_cost=trade_cost, + start=start, end=end) + strategy_analysis = StrategyAnalysis( + positions=backtesting_results["executors_df"], + candles_df=backtesting_results["processed_data"], + ) + metrics_container = BacktestingGraphs(backtesting_results["processed_data"]).get_trial_metrics(strategy_analysis, + add_positions=add_positions, + add_volume=add_volume) - market_data, positions = strategy.run_backtesting( - start='2021-04-01', - order_amount=50, - leverage=20, - initial_portfolio=100, - take_profit_multiplier=2.3, - stop_loss_multiplier=1.2, - time_limit=60 * 60 * 3, - std_span=None, - ) - strategy_analysis = StrategyAnalysis( - positions=positions, - candles_df=market_data, - ) - st.text(strategy_analysis.text_report()) - # TODO: check why the pnl is not being plotted - strategy_analysis.create_base_figure(volume=add_volume, positions=add_positions, trade_pnl=add_pnl) - st.plotly_chart(strategy_analysis.figure(), use_container_width=True) + except FileNotFoundError: + st.warning(f"The requested candles could not be found.") diff --git a/pages/launch_bot/README.md b/pages/launch_bot/README.md new file mode 100644 index 0000000..18f4d94 --- /dev/null +++ b/pages/launch_bot/README.md @@ -0,0 +1,19 @@ +### Description + +This page helps you deploy and manage Hummingbot instances: + +- Starting and stopping Hummingbot Broker +- Creating, starting and stopping bot instances +- Managing strategy and script files that instances run +- Fetching status of running instances + +### Maintainers + +This page is maintained by Hummingbot Foundation as a template other pages: + +* [cardosfede](https://github.com/cardosfede) +* [fengtality](https://github.com/fengtality) + +### Wiki + +See the [wiki](https://github.com/hummingbot/dashboard/wiki/%F0%9F%90%99-Bot-Orchestration) for more information. \ No newline at end of file diff --git a/pages/launch_bot/app.py b/pages/launch_bot/app.py new file mode 100644 index 0000000..dd2e43c --- /dev/null +++ b/pages/launch_bot/app.py @@ -0,0 +1,46 @@ +from types import SimpleNamespace + +import streamlit as st +from streamlit_elements import elements, mui + +from docker_manager import DockerManager + +from ui_components.dashboard import Dashboard +from ui_components.launch_strategy_v2 import LaunchStrategyV2 +from utils.st_utils import initialize_st_page + +CARD_WIDTH = 6 +CARD_HEIGHT = 3 +NUM_CARD_COLS = 2 + +initialize_st_page(title="Launch Bot", icon="πŸ™Œ", initial_sidebar_state="collapsed") + + +docker_manager = DockerManager() +if not docker_manager.is_docker_running(): + st.warning("Docker is not running. Please start Docker and refresh the page.") + st.stop() + + +def get_grid_positions(n_cards: int, cols: int = NUM_CARD_COLS, card_width: int = CARD_HEIGHT, card_height: int = CARD_WIDTH): + rows = n_cards // cols + 1 + x_y = [(x * card_width, y * card_height) for x in range(cols) for y in range(rows)] + return sorted(x_y, key=lambda x: (x[1], x[0])) + + +if "launch_bots_board" not in st.session_state: + board = Dashboard() + launch_bots_board = SimpleNamespace( + dashboard=board, + launch_bot=LaunchStrategyV2(board, 0, 0, 12, 10), + ) + st.session_state.launch_bots_board = launch_bots_board + +else: + launch_bots_board = st.session_state.launch_bots_board + + +with elements("create_bot"): + with mui.Paper(elevation=3, style={"padding": "2rem"}, spacing=[2, 2], container=True): + with launch_bots_board.dashboard(): + launch_bots_board.launch_bot() diff --git a/quants_lab/strategy/experiments/__init__.py b/quants_lab/controllers/__init__.py similarity index 100% rename from quants_lab/strategy/experiments/__init__.py rename to quants_lab/controllers/__init__.py diff --git a/quants_lab/controllers/bollinger.py b/quants_lab/controllers/bollinger.py new file mode 100644 index 0000000..de207fa --- /dev/null +++ b/quants_lab/controllers/bollinger.py @@ -0,0 +1,61 @@ +from typing import Optional + +import pandas as pd +import pandas_ta as ta +from hummingbot.smart_components.executors.position_executor.position_executor import PositionExecutor +from hummingbot.smart_components.strategy_frameworks.data_types import OrderLevel +from hummingbot.smart_components.strategy_frameworks.directional_trading import DirectionalTradingControllerConfigBase, \ + DirectionalTradingControllerBase +from pydantic import Field + + +class BollingerConf(DirectionalTradingControllerConfigBase): + strategy_name = "bollinger" + bb_length: int = Field(default=100, ge=2, le=1000) + bb_std: float = Field(default=2.0, ge=0.5, le=4.0) + bb_long_threshold: float = Field(default=0.0, ge=-3.0, le=0.5) + bb_short_threshold: float = Field(default=1.0, ge=0.5, le=3.0) + std_span: Optional[int] = Field(default=100, ge=100, le=400) + + +class Bollinger(DirectionalTradingControllerBase): + + def __init__(self, config: BollingerConf): + super().__init__(config) + self.config = config + + def early_stop_condition(self, executor: PositionExecutor, order_level: OrderLevel) -> bool: + """ + If an executor has an active position, should we close it based on a condition. + """ + return False + + def cooldown_condition(self, executor: PositionExecutor, order_level: OrderLevel) -> bool: + """ + After finishing an order, the executor will be in cooldown for a certain amount of time. + This prevents the executor from creating a new order immediately after finishing one and execute a lot + of orders in a short period of time from the same side. + """ + if executor.close_timestamp and executor.close_timestamp + order_level.cooldown_time > time.time(): + return True + return False + + def get_processed_data(self) -> pd.DataFrame: + df = self.candles[0].candles_df + + # Add indicators + df.ta.bbands(length=self.config.bb_length, std=self.config.bb_std, append=True) + + # Generate signal + long_condition = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] < self.config.bb_long_threshold + short_condition = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] > self.config.bb_short_threshold + + # Generate signal + df["signal"] = 0 + df.loc[long_condition, "signal"] = 1 + df.loc[short_condition, "signal"] = -1 + + # Optional: Generate spread multiplier + if self.config.std_span: + df["target"] = df["close"].rolling(self.config.std_span).std() / df["close"] + return df diff --git a/quants_lab/labeling/triple_barrier_method.py b/quants_lab/labeling/triple_barrier_method.py index d20c027..0650d56 100644 --- a/quants_lab/labeling/triple_barrier_method.py +++ b/quants_lab/labeling/triple_barrier_method.py @@ -60,20 +60,6 @@ def apply_tp_sl_on_tl(df: pd.DataFrame, tp: float, sl: float): stop_loss = pd.Series(index=df.index) # NaNs for loc, tl in events['tl'].fillna(df.index[-1]).items(): - # In the future we can think about including High and Low prices in the calculation - # side = events.at[loc, 'side'] # side (1 or -1) - # sl = stop_loss[loc] - # tp = take_profit[loc] - # close = df.close[loc] # path close price - # # path_close = df.close[loc:tl] # path prices - # path_high = (df.high[loc:tl] / close) - 1 # path high prices - # path_low = (df.low[loc:tl] / close) - 1 # path low prices - # if side == 1: - # df.loc[loc, 'stop_loss_time'] = path_low[path_low < sl].index.min() # earliest stop loss. - # df.loc[loc, 'take_profit_time'] = path_high[path_high > tp].index.min() # earliest profit taking. - # elif side == -1: - # df.loc[loc, 'stop_loss_time'] = path_high[path_high > -sl].index.min() - # df.loc[loc, 'take_profit_time'] = path_low[path_low < -tp].index.min() df0 = df.close[loc:tl] # path prices df0 = (df0 / df.close[loc] - 1) * events.at[loc, 'side'] # path returns df.loc[loc, 'stop_loss_time'] = df0[df0 < stop_loss[loc]].index.min() # earliest stop loss. diff --git a/quants_lab/strategy/experiments/bollinger.py b/quants_lab/strategy/experiments/bollinger.py deleted file mode 100644 index 9dd374e..0000000 --- a/quants_lab/strategy/experiments/bollinger.py +++ /dev/null @@ -1,36 +0,0 @@ -import pandas_ta as ta -from pydantic import BaseModel, Field - -from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase - - -class BollingerConf(BaseModel): - exchange: str = Field(default="binance_perpetual") - trading_pair: str = Field(default="ETH-USDT") - interval: str = Field(default="1h") - bb_length: int = Field(default=100, ge=2, le=1000) - bb_std: float = Field(default=2.0, ge=0.5, le=4.0) - bb_long_threshold: float = Field(default=0.0, ge=-3.0, le=0.5) - bb_short_threshold: float = Field(default=1.0, ge=0.5, le=3.0) - - -class Bollinger(DirectionalStrategyBase[BollingerConf]): - def get_raw_data(self): - df = self.get_candles( - exchange=self.config.exchange, - trading_pair=self.config.trading_pair, - interval=self.config.interval, - ) - return df - - def preprocessing(self, df): - df.ta.bbands(length=self.config.bb_length, std=self.config.bb_std, append=True) - return df - - def predict(self, df): - df["side"] = 0 - long_condition = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] < self.config.bb_long_threshold - short_condition = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] > self.config.bb_short_threshold - df.loc[long_condition, "side"] = 1 - df.loc[short_condition, "side"] = -1 - return df diff --git a/quants_lab/strategy/experiments/macd_bb.py b/quants_lab/strategy/experiments/macd_bb.py deleted file mode 100644 index a2ba82c..0000000 --- a/quants_lab/strategy/experiments/macd_bb.py +++ /dev/null @@ -1,45 +0,0 @@ -import pandas_ta as ta -from pydantic import BaseModel, Field - -from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase - - -class MACDBBConfig(BaseModel): - exchange: str = Field(default="binance_perpetual") - trading_pair: str = Field(default="ETH-USDT") - interval: str = Field(default="1h") - bb_length: int = Field(default=24, ge=2, le=1000) - bb_std: float = Field(default=2.0, ge=0.5, le=4.0) - bb_long_threshold: float = Field(default=0.0, ge=-3.0, le=0.5) - bb_short_threshold: float = Field(default=1.0, ge=0.5, le=3.0) - fast_macd: int = Field(default=21, ge=2, le=100) - slow_macd: int = Field(default=42, ge=30, le=1000) - signal_macd: int = Field(default=9, ge=2, le=100) - - -class MacdBollinger(DirectionalStrategyBase[MACDBBConfig]): - def get_raw_data(self): - df = self.get_candles( - exchange=self.config.exchange, - trading_pair=self.config.trading_pair, - interval=self.config.interval, - ) - return df - - def preprocessing(self, df): - df.ta.bbands(length=self.config.bb_length, std=self.config.bb_std, append=True) - df.ta.macd(fast=self.config.fast_macd, slow=self.config.slow_macd, signal=self.config.signal_macd, append=True) - return df - - def predict(self, df): - bbp = df[f"BBP_{self.config.bb_length}_{self.config.bb_std}"] - macdh = df[f"MACDh_{self.config.fast_macd}_{self.config.slow_macd}_{self.config.signal_macd}"] - macd = df[f"MACD_{self.config.fast_macd}_{self.config.slow_macd}_{self.config.signal_macd}"] - - long_condition = (bbp < self.config.bb_long_threshold) & (macdh > 0) & (macd < 0) - short_condition = (bbp > self.config.bb_short_threshold) & (macdh < 0) & (macd > 0) - - df["side"] = 0 - df.loc[long_condition, "side"] = 1 - df.loc[short_condition, "side"] = -1 - return df diff --git a/quants_lab/strategy/experiments/stat_arb.py b/quants_lab/strategy/experiments/stat_arb.py deleted file mode 100644 index a9060d4..0000000 --- a/quants_lab/strategy/experiments/stat_arb.py +++ /dev/null @@ -1,47 +0,0 @@ -import pandas as pd -import pandas_ta as ta -from pydantic import BaseModel, Field - -from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase - - -class StatArbConfig(BaseModel): - exchange: str = Field(default="binance_perpetual") - trading_pair: str = Field(default="ETH-USDT") - target_trading_pair: str = Field(default="BTC-USDT") - interval: str = Field(default="1h") - lookback: int = Field(default=100, ge=2, le=10000) - z_score_long: float = Field(default=2, ge=0, le=5) - z_score_short: float = Field(default=-2, ge=-5, le=0) - - -class StatArb(DirectionalStrategyBase[StatArbConfig]): - def get_raw_data(self): - df = self.get_candles( - exchange=self.config.exchange, - trading_pair=self.config.trading_pair, - interval=self.config.interval, - ) - df_target = self.get_candles( - exchange=self.config.exchange, - trading_pair=self.config.target_trading_pair, - interval=self.config.interval, - ) - df = pd.merge(df, df_target, on="timestamp", how='inner', suffixes=('', '_target')) - return df - - def preprocessing(self, df): - df["pct_change_original"] = df["close"].pct_change() - df["pct_change_target"] = df["close_target"].pct_change() - df["spread"] = df["pct_change_target"] - df["pct_change_original"] - df["cum_spread"] = df["spread"].rolling(self.config.lookback).sum() - df["z_score"] = ta.zscore(df["cum_spread"], length=self.config.lookback) - return df - - def predict(self, df): - df["side"] = 0 - short_condition = df["z_score"] < - self.config.z_score_short - long_condition = df["z_score"] > self.config.z_score_long - df.loc[long_condition, "side"] = 1 - df.loc[short_condition, "side"] = -1 - return df diff --git a/quants_lab/strategy/strategy_analysis.py b/quants_lab/strategy/strategy_analysis.py index f00267c..be57080 100644 --- a/quants_lab/strategy/strategy_analysis.py +++ b/quants_lab/strategy/strategy_analysis.py @@ -11,6 +11,9 @@ class StrategyAnalysis: def __init__(self, positions: pd.DataFrame, candles_df: Optional[pd.DataFrame] = None): self.candles_df = candles_df self.positions = positions + self.candles_df["timestamp"] = pd.to_datetime(self.candles_df["timestamp"], unit="ms") + self.positions["timestamp"] = pd.to_datetime(self.positions["timestamp"], unit="ms") + self.positions["close_time"] = pd.to_datetime(self.positions["close_time"], unit="ms") self.base_figure = None def create_base_figure(self, candlestick=True, volume=True, positions=False, trade_pnl=False, extra_rows=0): @@ -32,18 +35,18 @@ class StrategyAnalysis: def add_positions(self): # Add long and short positions active_signals = self.positions.copy() - active_signals.loc[active_signals['side'] == -1, 'symbol'] = 'triangle-down' - active_signals.loc[active_signals['side'] == 1, 'symbol'] = 'triangle-up' - active_signals.loc[active_signals['real_class'] == 1, 'color'] = 'lightgreen' - active_signals.loc[active_signals['real_class'] == -1, 'color'] = 'red' - self.base_figure.add_trace(go.Scatter(x=active_signals.loc[(active_signals['side'] != 0), 'timestamp'], - y=active_signals.loc[active_signals['side'] != 0, 'close'], - name='Entry Price: $', - mode='markers', - marker_color=active_signals.loc[(active_signals['side'] != 0), 'color'], - marker_symbol=active_signals.loc[(active_signals['side'] != 0), 'symbol'], + active_signals.loc[active_signals["signal"] == -1, "symbol"] = "triangle-down" + active_signals.loc[active_signals["signal"] == 1, "symbol"] = "triangle-up" + active_signals.loc[active_signals["profitable"] == 1, "color"] = "lightgreen" + active_signals.loc[active_signals["profitable"] == -1, "color"] = "red" + self.base_figure.add_trace(go.Scatter(x=active_signals.loc[(active_signals["side"] != 0), "timestamp"], + y=active_signals.loc[active_signals["side"] != 0, "close"], + name="Entry Price: $", + mode="markers", + marker_color=active_signals.loc[(active_signals["side"] != 0), "color"], + marker_symbol=active_signals.loc[(active_signals["side"] != 0), "symbol"], marker_size=20, - marker_line={'color': 'black', 'width': 0.7}), + marker_line={"color": "black", "width": 0.7}), row=1, col=1) for index, row in active_signals.iterrows(): @@ -53,7 +56,7 @@ class StrategyAnalysis: x0=row.timestamp, y0=row.close, x1=row.close_time, - y1=row.tp, + y1=row.take_profit_price, line=dict(color="green"), row=1, col=1) # Add SL @@ -63,7 +66,7 @@ class StrategyAnalysis: x0=row.timestamp, y0=row.close, x1=row.close_time, - y1=row.sl, + y1=row.stop_loss_price, line=dict(color="red"), row=1, col=1) @@ -81,11 +84,11 @@ class StrategyAnalysis: def add_candles_graph(self): self.base_figure.add_trace( go.Candlestick( - x=self.candles_df['timestamp'], - open=self.candles_df['open'], - high=self.candles_df['high'], - low=self.candles_df['low'], - close=self.candles_df['close'], + x=self.candles_df["timestamp"], + open=self.candles_df["open"], + high=self.candles_df["high"], + low=self.candles_df["low"], + close=self.candles_df["close"], name="OHLC" ), row=1, col=1, @@ -94,11 +97,11 @@ class StrategyAnalysis: def add_volume(self): self.base_figure.add_trace( go.Bar( - x=self.candles_df['timestamp'], - y=self.candles_df['volume'], + x=self.candles_df["timestamp"], + y=self.candles_df["volume"], name="Volume", opacity=0.5, - marker=dict(color='lightgreen') + marker=dict(color="lightgreen") ), row=2, col=1, ) @@ -106,23 +109,23 @@ class StrategyAnalysis: def add_trade_pnl(self, row=2): self.base_figure.add_trace( go.Scatter( - x=self.positions['timestamp'], - y=self.positions['ret_usd'].cumsum(), + x=self.positions["timestamp"], + y=self.positions["net_pnl_quote"].cumsum(), name="Cumulative Trade PnL", - mode='lines', - line=dict(color='chocolate', width=2)), + mode="lines", + line=dict(color="chocolate", width=2)), row=row, col=1 ) - self.base_figure.update_yaxes(title_text='Cum Trade PnL', row=row, col=1) + self.base_figure.update_yaxes(title_text="Cum Trade PnL", row=row, col=1) def update_layout(self, volume=True): self.base_figure.update_layout( title={ - 'text': "Backtesting Analysis", - 'y': 0.95, - 'x': 0.5, - 'xanchor': 'center', - 'yanchor': 'top' + "text": "Backtesting Analysis", + "y": 0.95, + "x": 0.5, + "xanchor": "center", + "yanchor": "top" }, legend=dict( orientation="h", @@ -131,9 +134,9 @@ class StrategyAnalysis: xanchor="right", x=1 ), - height=1000, + height=1500, xaxis_rangeslider_visible=False, - hovermode='x unified' + hovermode="x unified" ) self.base_figure.update_yaxes(title_text="Price", row=1, col=1) if volume: @@ -141,10 +144,10 @@ class StrategyAnalysis: self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) def initial_portfolio(self): - return self.positions['current_portfolio'].dropna().values[0] + return self.positions["inventory"].dropna().values[0] def final_portfolio(self): - return self.positions['current_portfolio'].dropna().values[-1] + return self.positions["inventory"].dropna().values[-1] def net_profit_usd(self): return self.final_portfolio() - self.initial_portfolio() @@ -153,22 +156,22 @@ class StrategyAnalysis: return self.net_profit_usd() / self.initial_portfolio() def returns(self): - return self.positions['ret_usd'] / self.initial_portfolio() + return self.positions["net_pnl_quote"] / self.initial_portfolio() def total_positions(self): return self.positions.shape[0] - 1 def win_signals(self): - return self.positions.loc[(self.positions['real_class'] > 0) & (self.positions["side"] != 0)] + return self.positions.loc[(self.positions["profitable"] > 0) & (self.positions["side"] != 0)] def loss_signals(self): - return self.positions.loc[(self.positions['real_class'] < 0) & (self.positions["side"] != 0)] + return self.positions.loc[(self.positions["profitable"] < 0) & (self.positions["side"] != 0)] def accuracy(self): return self.win_signals().shape[0] / self.total_positions() def max_drawdown_usd(self): - cumulative_returns = self.positions["ret_usd"].cumsum() + cumulative_returns = self.positions["net_pnl_quote"].cumsum() peak = np.maximum.accumulate(cumulative_returns) drawdown = (cumulative_returns - peak) max_draw_down = np.min(drawdown) @@ -182,25 +185,25 @@ class StrategyAnalysis: return returns.mean() / returns.std() def profit_factor(self): - total_won = self.win_signals().loc[:, 'ret_usd'].sum() - total_loss = - self.loss_signals().loc[:, 'ret_usd'].sum() + total_won = self.win_signals().loc[:, "net_pnl_quote"].sum() + total_loss = - self.loss_signals().loc[:, "net_pnl_quote"].sum() return total_won / total_loss def duration_in_minutes(self): - return (self.positions['timestamp'].iloc[-1] - self.positions['timestamp'].iloc[0]).total_seconds() / 60 + return (self.positions["timestamp"].iloc[-1] - self.positions["timestamp"].iloc[0]).total_seconds() / 60 def avg_trading_time_in_minutes(self): - time_diff_minutes = (pd.to_datetime(self.positions['close_time']) - self.positions['timestamp']).dt.total_seconds() / 60 + time_diff_minutes = (self.positions["close_time"] - self.positions["timestamp"]).dt.total_seconds() / 60 return time_diff_minutes.mean() def start_date(self): - return self.candles_df.timestamp.min() + return pd.to_datetime(self.candles_df.timestamp.min(), unit="ms") def end_date(self): - return self.candles_df.timestamp.max() + return pd.to_datetime(self.candles_df.timestamp.max(), unit="ms") def avg_profit(self): - return self.positions.ret_usd.mean() + return self.positions.net_pnl_quote.mean() def text_report(self): return f""" @@ -221,7 +224,7 @@ Strategy Performance Report: fig = go.Figure() fig.add_trace(go.Scatter(name="PnL Over Time", x=self.positions.index, - y=self.positions.ret_usd.cumsum())) + y=self.positions.net_pnl_quote.cumsum())) # Update layout with the required attributes fig.update_layout( title="PnL Over Time", diff --git a/ui_components/directional_strategies_file_explorer.py b/ui_components/directional_strategies_file_explorer.py index 8073972..2a93988 100644 --- a/ui_components/directional_strategies_file_explorer.py +++ b/ui_components/directional_strategies_file_explorer.py @@ -11,6 +11,6 @@ class DirectionalStrategiesFileExplorer(FileExplorerBase): onNodeSelect=lambda event, node_id: self.set_selected_file(event, node_id), defaultExpanded=["directional_strategies"]): with mui.lab.TreeItem(nodeId="directional_strategies", label=f"βš”οΈDirectional Strategies"): - strategies = get_python_files_from_directory(constants.DIRECTIONAL_STRATEGIES_PATH) + strategies = get_python_files_from_directory(constants.CONTROLLERS_PATH) for strategy in strategies: mui.lab.TreeItem(nodeId=strategy, label=f"🐍{strategy.split('/')[-1]}") diff --git a/ui_components/directional_strategy_creation_card.py b/ui_components/directional_strategy_creation_card.py index cc42777..9aa733d 100644 --- a/ui_components/directional_strategy_creation_card.py +++ b/ui_components/directional_strategy_creation_card.py @@ -1,7 +1,7 @@ from streamlit_elements import mui, lazy import constants -from utils.file_templates import directional_strategy_template +from utils.file_templates import directional_trading_controller_template from utils.os_utils import save_file from .dashboard import Dashboard @@ -20,9 +20,9 @@ class DirectionalStrategyCreationCard(Dashboard.Item): def _create_strategy(self): if self._strategy_type == "directional": - strategy_code = directional_strategy_template(self._strategy_name) + strategy_code = directional_trading_controller_template(self._strategy_name) save_file(name=f"{self._strategy_name.lower()}.py", content=strategy_code, - path=constants.DIRECTIONAL_STRATEGIES_PATH) + path=constants.CONTROLLERS_PATH) def __call__(self): with mui.Paper(key=self._key, sx={"display": "flex", "flexDirection": "column", "borderRadius": 3, "overflow": "hidden"}, elevation=1): diff --git a/ui_components/launch_master_bot_card.py b/ui_components/launch_master_bot_card.py index 6b6d1e5..07d0874 100644 --- a/ui_components/launch_master_bot_card.py +++ b/ui_components/launch_master_bot_card.py @@ -20,6 +20,8 @@ class LaunchMasterBotCard(Dashboard.Item): with st.spinner('Stopping Master Configs instance... This process may take a few seconds.'): time.sleep(5) else: + DockerManager().remove_container("hummingbot-master_bot_conf") + time.sleep(2) DockerManager().create_hummingbot_instance(instance_name="hummingbot-master_bot_conf", base_conf_folder="hummingbot_files/templates/master_bot_conf/.", target_conf_folder="hummingbot_files/templates/master_bot_conf/." @@ -41,31 +43,24 @@ class LaunchMasterBotCard(Dashboard.Item): mui.Typography("Master Configs", variant="h6", sx={"marginLeft": 1}) with mui.Grid(container=True, spacing=2, sx={"padding": "10px 15px 10px 15px"}): - with mui.Grid(item=True, xs=12): + with mui.Grid(item=True, xs=8): if not is_master_password_set: base_warning = "You need to set a master password in order to use the dashboard." if self.is_master_bot_running: mui.Alert(f"{base_warning} The Master Configs instance is running." - f" Attach to it in Terminal to set the master password.", severity="success") + f" Attach to it in Terminal to set the master password.", severity="success") else: mui.Alert(f"{base_warning} Master Configs instance isn't running. Start it and" - f" set the master password to continue.", severity="error") + f" set the master password to continue.", severity="error") else: if self.is_master_bot_running: mui.Alert("The Master Configs instance is running." - " Attach to it in Terminal to add credentials.", - severity="success", - sx={"margin": 2}) + " Attach to it in Terminal to add credentials.", + severity="success", + sx={"margin": 2}) else: mui.Alert("Master Configs instance isn't running. Start it to add credentials.", - severity="error") - - with mui.Grid(item=True, xs=8): - if self.is_master_bot_running: - mui.TextField(InputProps={"readOnly": True}, - label="Attach to Master Configs instance", - value="docker attach hummingbot-master_bot_conf", - sx={"width": "100%"}) + severity="error") with mui.Grid(item=True, xs=4): button_text = "Stop" if self.is_master_bot_running else "Start" color = "error" if self.is_master_bot_running else "success" @@ -75,4 +70,12 @@ class LaunchMasterBotCard(Dashboard.Item): variant="outlined", sx={"width": "100%", "height": "100%"}): icon() - mui.Typography(button_text) \ No newline at end of file + mui.Typography(button_text) + + with mui.Grid(item=True, xs=8): + if self.is_master_bot_running: + mui.TextField(InputProps={"readOnly": True}, + label="Attach to Master Configs instance", + value="docker attach hummingbot-master_bot_conf", + sx={"width": "100%"}) + diff --git a/ui_components/launch_strategy_v2.py b/ui_components/launch_strategy_v2.py new file mode 100644 index 0000000..4a68121 --- /dev/null +++ b/ui_components/launch_strategy_v2.py @@ -0,0 +1,138 @@ +import json +import os +import time + +from docker_manager import DockerManager +import streamlit as st +from hummingbot.core.data_type.common import PositionMode, OrderType, TradeType +from hummingbot.smart_components.utils import ConfigEncoderDecoder +from streamlit_elements import mui, lazy + +import constants +from utils.os_utils import get_directories_from_directory, get_python_files_from_directory, \ + get_yml_files_from_directory +from .dashboard import Dashboard + + +class LaunchStrategyV2(Dashboard.Item): + DEFAULT_ROWS = [] + DEFAULT_COLUMNS = DEFAULT_COLUMNS = [ + {"field": 'id', "headerName": 'ID', "width": 180}, + {"field": 'strategy_name', "headerName": 'Strategy Name', "width": 180, "editable": False, }, + {"field": 'exchange', "headerName": 'Exchange', "width": 180, "editable": True, }, + {"field": 'trading_pair', "headerName": 'Trading_pair', "width": 180, "editable": True, }, + ] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._controllers_available = get_python_files_from_directory(constants.CONTROLLERS_PATH) + self._controller_selected = self._controllers_available[0] + self._controller_configs_available = get_yml_files_from_directory("hummingbot_files/controller_configs") + self._controller_config_selected = None + self._bot_name = None + self._image_name = "hummingbot/hummingbot:latest" + self._base_bot_config = "master_bot_conf" + + def _set_bot_name(self, event): + self._bot_name = event.target.value + + def _set_image_name(self, event): + self._image_name = event.target.value + + def _set_base_bot_config(self, event): + self._base_bot_config = event.target.value + + def _set_controller(self, event): + self._controller_selected = event.target.value + + def _handle_row_selection(self, params, _): + self._controller_config_selected = params + + def launch_new_bot(self): + if self._bot_name and self._image_name and len(self._controller_config_selected) > 0: + bot_name = f"hummingbot-{self._bot_name}" + extra_environment_variables = ["-e", "CONFIG_FILE_NAME=strategy_v2_launcher.py", + "-e", f"controller_configs={','.join(self._controller_config_selected)}"] + DockerManager().create_hummingbot_instance(instance_name=bot_name, + base_conf_folder=f"{constants.HUMMINGBOT_TEMPLATES}/{self._base_bot_config}/.", + target_conf_folder=f"{constants.BOTS_FOLDER}/{bot_name}/.", + controllers_folder=constants.CONTROLLERS_PATH, + controllers_config_folder=constants.CONTROLLERS_CONFIG_PATH, + extra_environment_variables=extra_environment_variables, + image=self._image_name, + ) + with st.spinner('Starting Master Configs instance... This process may take a few seconds'): + time.sleep(3) + else: + st.warning("You need to define the bot name and select the controllers configs " + "that you want to deploy.") + + def __call__(self): + with mui.Paper(key=self._key, + sx={"display": "flex", "flexDirection": "column", "borderRadius": 3, "overflow": "hidden"}, + elevation=1): + with self.title_bar(padding="10px 15px 10px 15px", dark_switcher=False): + mui.Typography("πŸš€ Select the controller configs to launch", variant="h5") + + with mui.Grid(container=True, spacing=2, sx={"padding": "10px 15px 10px 15px"}): + with mui.Grid(item=True, xs=8): + mui.Alert( + "The new instance will contain the credentials configured in the following base instance:", + severity="info") + with mui.Grid(item=True, xs=4): + master_configs = [conf.split("/")[-2] for conf in + get_directories_from_directory(constants.HUMMINGBOT_TEMPLATES) if + "bot_conf" in conf] + with mui.FormControl(variant="standard", sx={"width": "100%"}): + mui.FormHelperText("Base Configs") + with mui.Select(label="Base Configs", defaultValue=master_configs[0], + variant="standard", onChange=lazy(self._set_base_bot_config)): + for master_config in master_configs: + mui.MenuItem(master_config, value=master_config) + with mui.Grid(item=True, xs=4): + mui.TextField(label="Instance Name", variant="outlined", onChange=lazy(self._set_bot_name), + sx={"width": "100%"}) + with mui.Grid(item=True, xs=4): + mui.TextField(label="Hummingbot Image", + defaultValue="hummingbot/hummingbot:latest", + variant="outlined", + placeholder="hummingbot-[name]", + onChange=lazy(self._set_image_name), + sx={"width": "100%"}) + with mui.Grid(item=True, xs=4): + with mui.Button(onClick=self.launch_new_bot, + variant="outlined", + color="success", + sx={"width": "100%", "height": "100%"}): + mui.icon.AddCircleOutline() + mui.Typography("Create") + + with mui.Grid(item=True, xs=8): + try: + encoder_decoder = ConfigEncoderDecoder(TradeType, OrderType, PositionMode) + data = [] + for config in self._controller_configs_available: + decoded_config = encoder_decoder.yaml_load(config) + data.append({"id": config.split("/")[-1], "strategy_name": decoded_config["strategy_name"], + "exchange": decoded_config["exchange"], "trading_pair": decoded_config["trading_pair"]}) + except json.JSONDecodeError: + data = self.DEFAULT_ROWS + + with mui.Paper(key=self._key, + sx={"display": "flex", "flexDirection": "column", "borderRadius": 3, + "overflow": "hidden", "height": 1000}, + elevation=1): + with self.title_bar(padding="10px 15px 10px 15px", dark_switcher=False): + mui.icon.ViewCompact() + mui.Typography("Data grid") + + with mui.Box(sx={"flex": 1, "minHeight": 3}): + mui.DataGrid( + columns=self.DEFAULT_COLUMNS, + rows=data, + pageSize=15, + rowsPerPageOptions=[15], + checkboxSelection=True, + disableSelectionOnClick=True, + onSelectionModelChange=self._handle_row_selection, + ) + diff --git a/ui_components/optimization_creation_card.py b/ui_components/optimization_creation_card.py index d224761..f6b99d1 100644 --- a/ui_components/optimization_creation_card.py +++ b/ui_components/optimization_creation_card.py @@ -3,7 +3,7 @@ import datetime import constants from utils.file_templates import strategy_optimization_template -from utils.os_utils import save_file, load_directional_strategies +from utils.os_utils import save_file, load_controllers from .dashboard import Dashboard @@ -27,7 +27,7 @@ class OptimizationCreationCard(Dashboard.Item): path=constants.OPTIMIZATIONS_PATH) def __call__(self): - available_strategies = load_directional_strategies(constants.DIRECTIONAL_STRATEGIES_PATH) + available_strategies = load_controllers(constants.CONTROLLERS_PATH) strategy_names = list(available_strategies.keys()) with mui.Paper(key=self._key, sx={"display": "flex", "flexDirection": "column", "borderRadius": 3, "overflow": "hidden"}, diff --git a/utils/file_templates.py b/utils/file_templates.py index 2ca4e81..bee0ac1 100644 --- a/utils/file_templates.py +++ b/utils/file_templates.py @@ -1,67 +1,90 @@ from typing import Dict -def directional_strategy_template(strategy_cls_name: str) -> str: +def directional_trading_controller_template(strategy_cls_name: str) -> str: strategy_config_cls_name = f"{strategy_cls_name}Config" sma_config_text = "{self.config.sma_length}" - return f"""import pandas_ta as ta -from pydantic import BaseModel, Field + return f"""import time +from typing import Optional -from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase +import pandas as pd +from pydantic import Field +from hummingbot.smart_components.executors.position_executor.position_executor import PositionExecutor +from hummingbot.smart_components.strategy_frameworks.data_types import OrderLevel +from hummingbot.smart_components.strategy_frameworks.directional_trading.directional_trading_controller_base import ( + DirectionalTradingControllerBase, + DirectionalTradingControllerConfigBase, +) -class {strategy_config_cls_name}(BaseModel): - name: str = "{strategy_cls_name.lower()}" - exchange: str = Field(default="binance_perpetual") - trading_pair: str = Field(default="ETH-USDT") - interval: str = Field(default="1h") +class {strategy_config_cls_name}(DirectionalTradingControllerConfigBase): + strategy_name: str = "{strategy_cls_name.lower()}" sma_length: int = Field(default=20, ge=10, le=200) # ... Add more fields here -class {strategy_cls_name}(DirectionalStrategyBase[{strategy_config_cls_name}]): +class {strategy_cls_name}(DirectionalTradingControllerBase): - def get_raw_data(self): - # The method get candles will search for the data in the folder data/candles - # If the data is not there, you can use the candles downloader to get the data - df = self.get_candles( - exchange=self.config.exchange, - trading_pair=self.config.trading_pair, - interval=self.config.interval, - ) - return df + def __init__(self, config: {strategy_config_cls_name}): + super().__init__(config) + self.config = config - def preprocessing(self, df): + + def early_stop_condition(self, executor: PositionExecutor, order_level: OrderLevel) -> bool: + # If an executor has an active position, should we close it based on a condition. This feature is not available + # for the backtesting yet + return False + + def cooldown_condition(self, executor: PositionExecutor, order_level: OrderLevel) -> bool: + # After finishing an order, the executor will be in cooldown for a certain amount of time. + # This prevents the executor from creating a new order immediately after finishing one and execute a lot + # of orders in a short period of time from the same side. + if executor.close_timestamp and executor.close_timestamp + order_level.cooldown_time > time.time(): + return True + return False + + def get_processed_data(self) -> pd.DataFrame: + df = self.candles[0].candles_df df.ta.sma(length=self.config.sma_length, append=True) # ... Add more indicators here # ... Check https://github.com/twopirllc/pandas-ta#indicators-by-category for more indicators # ... Use help(ta.indicator_name) to get more info - return df - def predict(self, df): # Generate long and short conditions long_cond = (df['close'] > df[f'SMA_{sma_config_text}']) short_cond = (df['close'] < df[f'SMA_{sma_config_text}']) # Choose side - df['side'] = 0 - df.loc[long_cond, 'side'] = 1 - df.loc[short_cond, 'side'] = -1 + df['signal'] = 0 + df.loc[long_cond, 'signal'] = 1 + df.loc[short_cond, 'signal'] = -1 return df """ def get_optuna_suggest_str(field_name: str, properties: Dict): - map_by_type = { - "number": "trial.suggest_float", - "integer": "trial.suggest_int", - "string": "trial.suggest_categorical", - } - config_num = f"('{field_name}', {properties.get('minimum', '_')}, {properties.get('maximum', '_')})" - config_cat = f"('{field_name}', ['{properties.get('default', '_')}',])" - optuna_trial_str = map_by_type[properties["type"]] + config_num if properties["type"] != "string" \ - else map_by_type[properties["type"]] + config_cat + if field_name == "candles_config": + return f"""{field_name}=[ + CandlesConfig(connector=exchange, trading_pair=trading_pair, + interval="3m", max_records=1000000) # Max number of candles for the real-time bot, + ]""" + if field_name == "strategy_name": + return f"{field_name}='{properties.get('default', '_')}'" + if field_name in ["order_levels", "trading_pair", "exchange"]: + return f"{field_name}={field_name}" + if field_name == "position_mode": + return f"{field_name}=PositionMode.HEDGE" + if field_name == "leverage": + return f"{field_name}=10" + if properties["type"] == "number": + optuna_trial_str = f"trial.suggest_float('{field_name}', {properties.get('minimum', '_')}, {properties.get('maximum', '_')}, step=0.01)" + elif properties["type"] == "integer": + optuna_trial_str = f"trial.suggest_int('{field_name}', {properties.get('minimum', '_')}, {properties.get('maximum', '_')})" + elif properties["type"] == "string": + optuna_trial_str = f"trial.suggest_categorical('{field_name}', ['{properties.get('default', '_')}',])" + else: + raise Exception(f"Unknown type {properties['type']} for field {field_name}") return f"{field_name}={optuna_trial_str}" @@ -73,47 +96,73 @@ def strategy_optimization_template(strategy_info: dict): fields_str = [get_optuna_suggest_str(field_name, properties) for field_name, properties in field_schema.items()] fields_str = "".join([f" {field_str},\n" for field_str in fields_str]) return f"""import traceback +from decimal import Decimal -from optuna import TrialPruned +from hummingbot.core.data_type.common import PositionMode, TradeType, OrderType +from hummingbot.data_feed.candles_feed.candles_factory import CandlesConfig +from hummingbot.smart_components.strategy_frameworks.data_types import TripleBarrierConf, OrderLevel +from hummingbot.smart_components.strategy_frameworks.directional_trading import DirectionalTradingBacktestingEngine +from hummingbot.smart_components.utils import ConfigEncoderDecoder +from optuna import TrialPruned -from quants_lab.strategy.experiments.{strategy_module} import {strategy_cls.__name__}, {strategy_config.__name__} -from quants_lab.strategy.strategy_analysis import StrategyAnalysis +from quants_lab.controllers.{strategy_module} import {strategy_cls.__name__}, {strategy_config.__name__} def objective(trial): try: + # General configuration for the backtesting + exchange = trial.suggest_categorical('exchange', ['binance_perpetual', ]) + trading_pair = trial.suggest_categorical('trading_pair', ['BTC-USDT', ]) + start = "2023-01-01" + end = "2023-08-01" + initial_portfolio_usd = 1000.0 + trade_cost = 0.0006 + + # The definition of order levels is not so necessary for directional strategies now but let's you customize the + # amounts for going long or short, the cooldown time between orders and the triple barrier configuration + stop_loss = trial.suggest_float('stop_loss', 0.01, 0.02, step=0.01) + take_profit = trial.suggest_float('take_profit', 0.01, 0.05, step=0.01) + time_limit = trial.suggest_int('time_limit', 60 * 60 * 2, 60 * 60 * 24) + + triple_barrier_conf = TripleBarrierConf( + stop_loss=Decimal(stop_loss), take_profit=Decimal(take_profit), + time_limit=time_limit, + trailing_stop_activation_price_delta=Decimal("0.008"), # It's not working yet with the backtesting engine + trailing_stop_trailing_delta=Decimal("0.004"), + ) + + order_levels = [ + OrderLevel(level=0, side=TradeType.BUY, order_amount_usd=Decimal(50), + cooldown_time=15, triple_barrier_conf=triple_barrier_conf), + OrderLevel(level=0, side=TradeType.SELL, order_amount_usd=Decimal(50), + cooldown_time=15, triple_barrier_conf=triple_barrier_conf), + ] config = {strategy_config.__name__}( {fields_str} ) - strategy = {strategy_cls.__name__}(config=config) - market_data, positions = strategy.run_backtesting( - start='2021-04-01', - order_amount=50, - leverage=20, - initial_portfolio=100, - take_profit_multiplier=trial.suggest_float("take_profit_multiplier", 1.0, 3.0), - stop_loss_multiplier=trial.suggest_float("stop_loss_multiplier", 1.0, 3.0), - time_limit=60 * 60 * trial.suggest_int("time_limit", 1, 24), - std_span=None, - ) - strategy_analysis = StrategyAnalysis( - positions=positions, - ) - - trial.set_user_attr("net_profit_usd", strategy_analysis.net_profit_usd()) - trial.set_user_attr("net_profit_pct", strategy_analysis.net_profit_pct()) - trial.set_user_attr("max_drawdown_usd", strategy_analysis.max_drawdown_usd()) - trial.set_user_attr("max_drawdown_pct", strategy_analysis.max_drawdown_pct()) - trial.set_user_attr("sharpe_ratio", strategy_analysis.sharpe_ratio()) - trial.set_user_attr("accuracy", strategy_analysis.accuracy()) - trial.set_user_attr("total_positions", strategy_analysis.total_positions()) - trial.set_user_attr("win_signals", strategy_analysis.win_signals().shape[0]) - trial.set_user_attr("loss_signals", strategy_analysis.loss_signals().shape[0]) - trial.set_user_attr("profit_factor", strategy_analysis.profit_factor()) - trial.set_user_attr("duration_in_hours", strategy_analysis.duration_in_minutes() / 60) - trial.set_user_attr("avg_trading_time_in_hours", strategy_analysis.avg_trading_time_in_minutes() / 60) - trial.set_user_attr("config", config.dict()) - return strategy_analysis.net_profit_pct() + controller = {strategy_cls.__name__}(config=config) + engine = DirectionalTradingBacktestingEngine(controller=controller) + engine.load_controller_data("./data/candles") + backtesting_results = engine.run_backtesting(initial_portfolio_usd=initial_portfolio_usd, trade_cost=trade_cost, + start=start, end=end) + + strategy_analysis = backtesting_results["results"] + encoder_decoder = ConfigEncoderDecoder(TradeType, OrderType, PositionMode) + + trial.set_user_attr("net_pnl_quote", strategy_analysis["net_pnl_quote"]) + trial.set_user_attr("net_pnl_pct", strategy_analysis["net_pnl"]) + trial.set_user_attr("max_drawdown_usd", strategy_analysis["max_drawdown_usd"]) + trial.set_user_attr("max_drawdown_pct", strategy_analysis["max_drawdown_pct"]) + trial.set_user_attr("sharpe_ratio", strategy_analysis["sharpe_ratio"]) + trial.set_user_attr("accuracy", strategy_analysis["accuracy"]) + trial.set_user_attr("total_positions", strategy_analysis["total_positions"]) + trial.set_user_attr("profit_factor", strategy_analysis["profit_factor"]) + trial.set_user_attr("duration_in_hours", strategy_analysis["duration_minutes"] / 60) + trial.set_user_attr("avg_trading_time_in_hours", strategy_analysis["avg_trading_time_minutes"] / 60) + trial.set_user_attr("win_signals", strategy_analysis["win_signals"]) + trial.set_user_attr("loss_signals", strategy_analysis["loss_signals"]) + trial.set_user_attr("config", encoder_decoder.encode(config.dict())) + return strategy_analysis["net_pnl"] except Exception as e: traceback.print_exc() raise TrialPruned() diff --git a/utils/graphs.py b/utils/graphs.py index c4b9b2b..f090978 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -242,7 +242,7 @@ class BacktestingGraphs: fig = go.Figure() fig.add_trace(go.Scatter(name="Pnl vs Max Drawdown", x=-100 * self.study_df["max_drawdown_pct"], - y=100 * self.study_df["net_profit_pct"], + y=100 * self.study_df["net_pnl_pct"], mode="markers", text=None, hovertext=self.study_df["hover_text"])) diff --git a/utils/optuna_database_manager.py b/utils/optuna_database_manager.py index 5c79c18..d3a68a3 100644 --- a/utils/optuna_database_manager.py +++ b/utils/optuna_database_manager.py @@ -225,8 +225,8 @@ class OptunaDBManager: f"Loss Signals: {x['loss_signals']}
" f"Max Drawdown [%]: {100 * x['max_drawdown_pct']:.2f} %
" f"Max Drawdown [USD]: $ {x['max_drawdown_usd']:.2f}
" - f"Net Profit [%]: {100 * x['net_profit_pct']:.2f} %
" - f"Net Profit [$]: $ {x['net_profit_usd']:.2f}
" + f"Net Profit [%]: {100 * x['net_pnl_pct']:.2f} %
" + f"Net Profit [$]: $ {x['net_pnl_quote']:.2f}
" f"Profit Factor: {x['profit_factor']:.2f}
" f"Sharpe Ratio: {x['sharpe_ratio']:.4f}
" f"Total Positions: {x['total_positions']}
" @@ -238,7 +238,7 @@ class OptunaDBManager: def _get_merged_df(self): float_cols = ["accuracy", "avg_trading_time_in_hours", "duration_in_hours", "max_drawdown_pct", "max_drawdown_usd", - "net_profit_pct", "net_profit_usd", "profit_factor", "sharpe_ratio", "value"] + "net_pnl_pct", "net_pnl_quote", "profit_factor", "sharpe_ratio", "value"] int_cols = ["loss_signals", "total_positions", "win_signals"] merged_df = self.trials\ .merge(self.studies, on="study_id")\ diff --git a/utils/os_utils.py b/utils/os_utils.py index 6dbf01a..abb0863 100644 --- a/utils/os_utils.py +++ b/utils/os_utils.py @@ -3,10 +3,8 @@ import subprocess import importlib.util import inspect import os +from hummingbot.smart_components.strategy_frameworks.directional_trading import DirectionalTradingControllerBase, DirectionalTradingControllerConfigBase -from pydantic import BaseModel - -from quants_lab.strategy.directional_strategy_base import DirectionalStrategyBase # update this to the actual import import yaml @@ -14,9 +12,11 @@ def remove_files_from_directory(directory: str): for file in os.listdir(directory): os.remove(f"{directory}/{file}") + def remove_file(file_path: str): os.remove(file_path) + def remove_directory(directory: str): process = subprocess.Popen(f"rm -rf {directory}", shell=True) process.wait() @@ -78,22 +78,22 @@ def get_yml_files_from_directory(directory: str) -> list: return yml -def load_directional_strategies(path): - strategies = {} +def load_controllers(path): + controllers = {} for filename in os.listdir(path): if filename.endswith('.py') and "__init__" not in filename: module_name = filename[:-3] # strip the .py to get the module name - strategies[module_name] = {"module": module_name} + controllers[module_name] = {"module": module_name} file_path = os.path.join(path, filename) spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) for name, cls in inspect.getmembers(module, inspect.isclass): - if issubclass(cls, DirectionalStrategyBase) and cls is not DirectionalStrategyBase: - strategies[module_name]["class"] = cls - if issubclass(cls, BaseModel) and cls is not BaseModel: - strategies[module_name]["config"] = cls - return strategies + if issubclass(cls, DirectionalTradingControllerBase) and cls is not DirectionalTradingControllerBase: + controllers[module_name]["class"] = cls + if issubclass(cls, DirectionalTradingControllerConfigBase) and cls is not DirectionalTradingControllerConfigBase: + controllers[module_name]["config"] = cls + return controllers def get_function_from_file(file_path: str, function_name: str):