From 3d430cb65d76a3b01e0cd8b7ae8f8caa976a4f2b Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 18:55:30 -0300 Subject: [PATCH 01/14] (fix) select correct environment --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c2d12e0..14be7c3 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ env_remove: conda env remove -n dashboard env_create: - conda env create -f environment.yml + conda env create -f environment_conda.yml docker_build: docker build -t dashboard:latest . From 69936f3f6fe400bb21b86ea6e22e1dc030c1c415 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 18:55:41 -0300 Subject: [PATCH 02/14] (feat) add pyyaml --- environment_conda.yml | 3 ++- requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/environment_conda.yml b/environment_conda.yml index fcb3c61..50e5c45 100644 --- a/environment_conda.yml +++ b/environment_conda.yml @@ -14,4 +14,5 @@ dependencies: - glom - defillama - statsmodels - - pandas_ta \ No newline at end of file + - pandas_ta + - pyyaml diff --git a/requirements.txt b/requirements.txt index 3d01a8c..eda1c5c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ - defillama - statsmodels - pandas_ta -- sqlalchemy \ No newline at end of file +- sqlalchemy +- pyyaml \ No newline at end of file From 72df9185f820e8a7cc86654d54e5bf6862a0cf7d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 19:43:07 -0300 Subject: [PATCH 03/14] (feat) add os_utils --- utils/os_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 utils/os_utils.py diff --git a/utils/os_utils.py b/utils/os_utils.py new file mode 100644 index 0000000..4456ef2 --- /dev/null +++ b/utils/os_utils.py @@ -0,0 +1,12 @@ +import os +import yaml + + +def remove_files_from_directory(directory: str): + for file in os.listdir(directory): + os.remove(f"{directory}/{file}") + + +def dump_dict_to_yaml(data_dict, filename): + with open(filename, 'w') as file: + yaml.dump(data_dict, file) From 093876ae526b05608025459f100992c43a135060 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 19:43:26 -0300 Subject: [PATCH 04/14] (feat) add docker_manager.py --- utils/docker_manager.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 utils/docker_manager.py diff --git a/utils/docker_manager.py b/utils/docker_manager.py new file mode 100644 index 0000000..1bf7d27 --- /dev/null +++ b/utils/docker_manager.py @@ -0,0 +1,41 @@ +import subprocess +from typing import Dict + +import constants +from utils import os_utils + + +class DockerManager: + def __init__(self): + pass + + @staticmethod + def get_active_containers(): + cmd = "docker ps --format '{{.Names}}'" + output = subprocess.check_output(cmd, shell=True) + backtestings = [container for container in output.decode().split()] + return backtestings + + @staticmethod + def get_exited_containers(): + cmd = "docker ps --filter status=exited --format '{{.Names}}'" + output = subprocess.check_output(cmd, shell=True) + containers = output.decode().split() + return containers + + @staticmethod + def clean_exited_containers(): + cmd = "docker container prune --force" + subprocess.Popen(cmd, shell=True) + + def stop_active_containers(self): + containers = self.get_active_containers() + for container in containers: + cmd = f"docker stop {container}" + subprocess.Popen(cmd, shell=True) + + def create_download_candles_container(self, candles_config: Dict): + os_utils.dump_dict_to_yaml(candles_config, constants.DOWNLOAD_CANDLES_CONFIG_YML) + command = ["docker", "compose", "-p", "data_downloader", "-f", + "hummingbot_files/compose_files/data-downloader-compose.yml", "up", "-d"] + subprocess.Popen(command) From 16a857314ebbc64cd61dd4e9f2526078be0e4b2f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 19:45:45 -0300 Subject: [PATCH 05/14] (feat) add constants --- constants.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 constants.py diff --git a/constants.py b/constants.py new file mode 100644 index 0000000..7336e6e --- /dev/null +++ b/constants.py @@ -0,0 +1,2 @@ +CANDLES_DATA_PATH = "data/candles" +DOWNLOAD_CANDLES_CONFIG_YML = "hummingbot_files/scripts_configs/data_downloader_config.yml" From d1d98ca4a870dea922a6fad83e97fb4ffd819aaf Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 19:46:08 -0300 Subject: [PATCH 06/14] (feat) add hummingbot files for startup --- hummingbot_files/conf/.password_verification | 1 + hummingbot_files/conf/conf_client.yml | 194 ++++++++++++ hummingbot_files/conf/conf_fee_overrides.yml | 298 +++++++++++++++++++ hummingbot_files/conf/hummingbot_logs.yml | 83 ++++++ 4 files changed, 576 insertions(+) create mode 100644 hummingbot_files/conf/.password_verification create mode 100644 hummingbot_files/conf/conf_client.yml create mode 100644 hummingbot_files/conf/conf_fee_overrides.yml create mode 100755 hummingbot_files/conf/hummingbot_logs.yml diff --git a/hummingbot_files/conf/.password_verification b/hummingbot_files/conf/.password_verification new file mode 100644 index 0000000..b8c7618 --- /dev/null +++ b/hummingbot_files/conf/.password_verification @@ -0,0 +1 @@ +7b2263727970746f223a207b22636970686572223a20226165732d3132382d637472222c2022636970686572706172616d73223a207b226976223a20223864336365306436393461623131396334363135663935366464653839363063227d2c202263697068657274657874223a20223836333266323430613563306131623665353664222c20226b6466223a202270626b646632222c20226b6466706172616d73223a207b2263223a20313030303030302c2022646b6c656e223a2033322c2022707266223a2022686d61632d736861323536222c202273616c74223a20226566373330376531636464373964376132303338323534656139343433663930227d2c20226d6163223a202266393439383534613530633138363633386363353962336133363665633962353333386633613964373266636635343066313034333361353431636232306438227d2c202276657273696f6e223a20337d \ No newline at end of file diff --git a/hummingbot_files/conf/conf_client.yml b/hummingbot_files/conf/conf_client.yml new file mode 100644 index 0000000..dc0b28a --- /dev/null +++ b/hummingbot_files/conf/conf_client.yml @@ -0,0 +1,194 @@ +#################################### +### client_config_map config ### +#################################### + +instance_id: 039758736d451914503a45ff596e168902d62557 + +log_level: INFO + +debug_console: false + +strategy_report_interval: 900.0 + +logger_override_whitelist: +- hummingbot.strategy.arbitrage +- hummingbot.strategy.cross_exchange_market_making +- conf + +log_file_path: /home/hummingbot/logs + +kill_switch_mode: {} + +# What to auto-fill in the prompt after each import command (start/config) +autofill_import: disabled + +telegram_mode: {} + +# MQTT Bridge configuration. +mqtt_bridge: + mqtt_host: localhost + mqtt_port: 1883 + mqtt_username: '' + mqtt_password: '' + mqtt_namespace: hbot + mqtt_ssl: false + mqtt_logger: true + mqtt_notifier: true + mqtt_commands: true + mqtt_events: true + mqtt_external_events: true + mqtt_autostart: false + +# Error log sharing +send_error_logs: true + +# Can store the previous strategy ran for quick retrieval. +previous_strategy: null + +# Advanced database options, currently supports SQLAlchemy's included dialects +# Reference: https://docs.sqlalchemy.org/en/13/dialects/ +# To use an instance of SQLite DB the required configuration is +# db_engine: sqlite +# To use a DBMS the required configuration is +# db_host: 127.0.0.1 +# db_port: 3306 +# db_username: username +# db_password: password +# db_name: dbname +db_mode: + db_engine: sqlite + +pmm_script_mode: {} + +# Balance Limit Configurations +# e.g. Setting USDT and BTC limits on Binance. +# balance_asset_limit: +# binance: +# BTC: 0.1 +# USDT: 1000 +balance_asset_limit: + bybit_testnet: {} + lbank: {} + binance_us: {} + crypto_com: {} + ascend_ex_paper_trade: {} + hotbit: {} + gate_io_paper_trade: {} + bitmex_testnet: {} + ndax_testnet: {} + huobi: {} + probit_kr: {} + altmarkets: {} + hitbtc: {} + foxbit: {} + ascend_ex: {} + binance: {} + okx: {} + ciex: {} + bitmex: {} + bitfinex: {} + probit: {} + kraken: {} + kucoin: {} + bitmart: {} + bybit: {} + bittrex: {} + btc_markets: {} + mock_paper_exchange: {} + kucoin_paper_trade: {} + ndax: {} + loopring: {} + mexc: {} + whitebit: {} + coinbase_pro: {} + binance_paper_trade: {} + gate_io: {} + +# Fixed gas price (in Gwei) for Ethereum transactions +manual_gas_price: 50.0 + +# Gateway API Configurations +# default host to only use localhost +# Port need to match the final installation port for Gateway +gateway: + gateway_api_host: localhost + gateway_api_port: '15888' + +certs_path: /home/hummingbot/certs + +# Whether to enable aggregated order and trade data collection +anonymized_metrics_mode: + anonymized_metrics_interval_min: 15.0 + +# Command Shortcuts +# Define abbreviations for often used commands +# or batch grouped commands together +command_shortcuts: +- command: spreads + help: Set bid and ask spread + arguments: + - Bid Spread + - Ask Spread + output: + - config bid_spread $1 + - config ask_spread $2 + +# A source for rate oracle, currently ascend_ex, binance, coin_gecko, kucoin, gate_io +rate_oracle_source: + name: binance + +# A universal token which to display tokens values in, e.g. USD,EUR,BTC +global_token: + global_token_name: USD + global_token_symbol: $ + +# Percentage of API rate limits (on any exchange and any end point) allocated to this bot instance. +# Enter 50 to indicate 50%. E.g. if the API rate limit is 100 calls per second, and you allocate +# 50% to this setting, the bot will have a maximum (limit) of 50 calls per second +rate_limits_share_pct: 100.0 + +commands_timeout: + create_command_timeout: 10.0 + other_commands_timeout: 30.0 + +# Tabulate table format style (https://github.com/astanin/python-tabulate#table-format) +tables_format: psql + +paper_trade: + paper_trade_exchanges: + - binance + - kucoin + - ascend_ex + - gate_io + paper_trade_account_balance: + BTC: 1.0 + USDT: 1000.0 + ONE: 1000.0 + USDQ: 1000.0 + TUSD: 1000.0 + ETH: 10.0 + WETH: 10.0 + USDC: 1000.0 + DAI: 1000.0 + +color: + top_pane: '#000000' + bottom_pane: '#000000' + output_pane: '#262626' + input_pane: '#1C1C1C' + logs_pane: '#121212' + terminal_primary: '#5FFFD7' + primary_label: '#5FFFD7' + secondary_label: '#FFFFFF' + success_label: '#5FFFD7' + warning_label: '#FFFF00' + info_label: '#5FD7FF' + error_label: '#FF0000' + gold_label: '#FFD700' + silver_label: '#C0C0C0' + bronze_label: '#CD7F32' + +# The tick size is the frequency with which the clock notifies the time iterators by calling the +# c_tick() method, that means for example that if the tick size is 1, the logic of the strategy +# will run every second. +tick_size: 1.0 diff --git a/hummingbot_files/conf/conf_fee_overrides.yml b/hummingbot_files/conf/conf_fee_overrides.yml new file mode 100644 index 0000000..091bc0e --- /dev/null +++ b/hummingbot_files/conf/conf_fee_overrides.yml @@ -0,0 +1,298 @@ +######################################## +### Fee overrides configurations ### +######################################## + +# For more detailed information: https://docs.hummingbot.io +template_version: 14 + +# Example of the fields that can be specified to override the `TradeFeeFactory` default settings. +# If the field is missing or the value is left blank, the default value will be used. +# The percentage values are specified as 0.1 for 0.1%. +# +# [exchange name]_percent_fee_token: +# [exchange name]_maker_percent_fee: +# [exchange name]_taker_percent_fee: +# [exchange name]_buy_percent_fee_deducted_from_returns: # if False, the buy fee is added to the order costs +# [exchange name]_maker_fixed_fees: # a list of lists of token-fee pairs (e.g. [["ETH", 1]]) +# [exchange name]_taker_fixed_fees: # a list of lists of token-fee pairs (e.g. [["ETH", 1]]) + +binance_percent_fee_token: # BNB +binance_maker_percent_fee: # 0.75 +binance_taker_percent_fee: # 0.75 +binance_buy_percent_fee_deducted_from_returns: # True + +# List of supported Exchanges for which the user's conf/conf_fee_override.yml +# will work. This file currently needs to be in sync with hummingbot list of +# supported exchanges +altmarkets_buy_percent_fee_deducted_from_returns: +altmarkets_maker_fixed_fees: +altmarkets_maker_percent_fee: +altmarkets_percent_fee_token: +altmarkets_taker_fixed_fees: +altmarkets_taker_percent_fee: +ascend_ex_buy_percent_fee_deducted_from_returns: +ascend_ex_maker_fixed_fees: +ascend_ex_maker_percent_fee: +ascend_ex_percent_fee_token: +ascend_ex_taker_fixed_fees: +ascend_ex_taker_percent_fee: +binance_maker_fixed_fees: +binance_perpetual_buy_percent_fee_deducted_from_returns: +binance_perpetual_maker_fixed_fees: +binance_perpetual_maker_percent_fee: +binance_perpetual_percent_fee_token: +binance_perpetual_taker_fixed_fees: +binance_perpetual_taker_percent_fee: +binance_perpetual_testnet_buy_percent_fee_deducted_from_returns: +binance_perpetual_testnet_maker_fixed_fees: +binance_perpetual_testnet_maker_percent_fee: +binance_perpetual_testnet_percent_fee_token: +binance_perpetual_testnet_taker_fixed_fees: +binance_perpetual_testnet_taker_percent_fee: +binance_taker_fixed_fees: +binance_us_buy_percent_fee_deducted_from_returns: +binance_us_maker_fixed_fees: +binance_us_maker_percent_fee: +binance_us_percent_fee_token: +binance_us_taker_fixed_fees: +binance_us_taker_percent_fee: +bitfinex_buy_percent_fee_deducted_from_returns: +bitfinex_maker_fixed_fees: +bitfinex_maker_percent_fee: +bitfinex_percent_fee_token: +bitfinex_taker_fixed_fees: +bitfinex_taker_percent_fee: +bitmart_buy_percent_fee_deducted_from_returns: +bitmart_maker_fixed_fees: +bitmart_maker_percent_fee: +bitmart_percent_fee_token: +bitmart_taker_fixed_fees: +bitmart_taker_percent_fee: +bittrex_buy_percent_fee_deducted_from_returns: +bittrex_maker_fixed_fees: +bittrex_maker_percent_fee: +bittrex_percent_fee_token: +bittrex_taker_fixed_fees: +bittrex_taker_percent_fee: +btc_markets_percent_fee_token: +btc_markets_maker_percent_fee: +btc_markets_taker_percent_fee: +btc_markets_buy_percent_fee_deducted_from_returns: +bybit_perpetual_buy_percent_fee_deducted_from_returns: +bybit_perpetual_maker_fixed_fees: +bybit_perpetual_maker_percent_fee: +bybit_perpetual_percent_fee_token: +bybit_perpetual_taker_fixed_fees: +bybit_perpetual_taker_percent_fee: +bybit_perpetual_testnet_buy_percent_fee_deducted_from_returns: +bybit_perpetual_testnet_maker_fixed_fees: +bybit_perpetual_testnet_maker_percent_fee: +bybit_perpetual_testnet_percent_fee_token: +bybit_perpetual_testnet_taker_fixed_fees: +bybit_perpetual_testnet_taker_percent_fee: +coinbase_pro_buy_percent_fee_deducted_from_returns: +coinbase_pro_maker_fixed_fees: +coinbase_pro_maker_percent_fee: +coinbase_pro_percent_fee_token: +coinbase_pro_taker_fixed_fees: +coinbase_pro_taker_percent_fee: +crypto_com_buy_percent_fee_deducted_from_returns: +crypto_com_maker_fixed_fees: +crypto_com_maker_percent_fee: +crypto_com_percent_fee_token: +crypto_com_taker_fixed_fees: +crypto_com_taker_percent_fee: +dydx_perpetual_buy_percent_fee_deducted_from_returns: +dydx_perpetual_maker_fixed_fees: +dydx_perpetual_maker_percent_fee: +dydx_perpetual_percent_fee_token: +dydx_perpetual_taker_fixed_fees: +dydx_perpetual_taker_percent_fee: +gate_io_buy_percent_fee_deducted_from_returns: +gate_io_maker_fixed_fees: +gate_io_maker_percent_fee: +gate_io_percent_fee_token: +gate_io_taker_fixed_fees: +gate_io_taker_percent_fee: +hitbtc_buy_percent_fee_deducted_from_returns: +hitbtc_maker_fixed_fees: +hitbtc_maker_percent_fee: +hitbtc_percent_fee_token: +hitbtc_taker_fixed_fees: +hitbtc_taker_percent_fee: +huobi_buy_percent_fee_deducted_from_returns: +huobi_maker_fixed_fees: +huobi_maker_percent_fee: +huobi_percent_fee_token: +huobi_taker_fixed_fees: +huobi_taker_percent_fee: +kraken_buy_percent_fee_deducted_from_returns: +kraken_maker_fixed_fees: +kraken_maker_percent_fee: +kraken_percent_fee_token: +kraken_taker_fixed_fees: +kraken_taker_percent_fee: +kucoin_buy_percent_fee_deducted_from_returns: +kucoin_maker_fixed_fees: +kucoin_maker_percent_fee: +kucoin_percent_fee_token: +kucoin_taker_fixed_fees: +kucoin_taker_percent_fee: +loopring_buy_percent_fee_deducted_from_returns: +loopring_maker_fixed_fees: +loopring_maker_percent_fee: +loopring_percent_fee_token: +loopring_taker_fixed_fees: +loopring_taker_percent_fee: +mexc_buy_percent_fee_deducted_from_returns: +mexc_maker_fixed_fees: +mexc_maker_percent_fee: +mexc_percent_fee_token: +mexc_taker_fixed_fees: +mexc_taker_percent_fee: +ndax_buy_percent_fee_deducted_from_returns: +ndax_maker_fixed_fees: +ndax_maker_percent_fee: +ndax_percent_fee_token: +ndax_taker_fixed_fees: +ndax_taker_percent_fee: +ndax_testnet_buy_percent_fee_deducted_from_returns: +ndax_testnet_maker_fixed_fees: +ndax_testnet_maker_percent_fee: +ndax_testnet_percent_fee_token: +ndax_testnet_taker_fixed_fees: +ndax_testnet_taker_percent_fee: +okx_buy_percent_fee_deducted_from_returns: +okx_maker_fixed_fees: +okx_maker_percent_fee: +okx_percent_fee_token: +okx_taker_fixed_fees: +okx_taker_percent_fee: +probit_buy_percent_fee_deducted_from_returns: +probit_kr_buy_percent_fee_deducted_from_returns: +probit_kr_maker_fixed_fees: +probit_kr_maker_percent_fee: +probit_kr_percent_fee_token: +probit_kr_taker_fixed_fees: +probit_kr_taker_percent_fee: +probit_maker_fixed_fees: +probit_maker_percent_fee: +probit_percent_fee_token: +probit_taker_fixed_fees: +probit_taker_percent_fee: +bitmex_perpetual_percent_fee_token: +bitmex_perpetual_maker_percent_fee: +bitmex_perpetual_taker_percent_fee: +bitmex_perpetual_buy_percent_fee_deducted_from_returns: +bitmex_perpetual_maker_fixed_fees: +bitmex_perpetual_taker_fixed_fees: +bitmex_perpetual_testnet_percent_fee_token: +bitmex_perpetual_testnet_maker_percent_fee: +bitmex_perpetual_testnet_taker_percent_fee: +bitmex_perpetual_testnet_buy_percent_fee_deducted_from_returns: +bitmex_perpetual_testnet_maker_fixed_fees: +bitmex_perpetual_testnet_taker_fixed_fees: +kucoin_perpetual_percent_fee_token: +kucoin_perpetual_maker_percent_fee: +kucoin_perpetual_taker_percent_fee: +kucoin_perpetual_buy_percent_fee_deducted_from_returns: +kucoin_perpetual_maker_fixed_fees: +kucoin_perpetual_taker_fixed_fees: +kucoin_perpetual_testnet_percent_fee_token: +kucoin_perpetual_testnet_maker_percent_fee: +kucoin_perpetual_testnet_taker_percent_fee: +kucoin_perpetual_testnet_buy_percent_fee_deducted_from_returns: +kucoin_perpetual_testnet_maker_fixed_fees: +kucoin_perpetual_testnet_taker_fixed_fees: +gate_io_perpetual_percent_fee_token: +gate_io_perpetual_maker_percent_fee: +gate_io_perpetual_taker_percent_fee: +gate_io_perpetual_buy_percent_fee_deducted_from_returns: +gate_io_perpetual_maker_fixed_fees: +gate_io_perpetual_taker_fixed_fees: +phemex_perpetual_percent_fee_token: +phemex_perpetual_maker_percent_fee: +phemex_perpetual_taker_percent_fee: +phemex_perpetual_buy_percent_fee_deducted_from_returns: +phemex_perpetual_maker_fixed_fees: +phemex_perpetual_taker_fixed_fees: +phemex_perpetual_testnet_percent_fee_token: +phemex_perpetual_testnet_maker_percent_fee: +phemex_perpetual_testnet_taker_percent_fee: +phemex_perpetual_testnet_buy_percent_fee_deducted_from_returns: +phemex_perpetual_testnet_maker_fixed_fees: +phemex_perpetual_testnet_taker_fixed_fees: +bitget_perpetual_percent_fee_token: +bitget_perpetual_maker_percent_fee: +bitget_perpetual_taker_percent_fee: +bitget_perpetual_buy_percent_fee_deducted_from_returns: +bitget_perpetual_maker_fixed_fees: +bitget_perpetual_taker_fixed_fees: +bit_com_perpetual_percent_fee_token: +bit_com_perpetual_maker_percent_fee: +bit_com_perpetual_taker_percent_fee: +bit_com_perpetual_buy_percent_fee_deducted_from_returns: +bit_com_perpetual_maker_fixed_fees: +bit_com_perpetual_taker_fixed_fees: +bit_com_perpetual_testnet_percent_fee_token: +bit_com_perpetual_testnet_maker_percent_fee: +bit_com_perpetual_testnet_taker_percent_fee: +bit_com_perpetual_testnet_buy_percent_fee_deducted_from_returns: +bit_com_perpetual_testnet_maker_fixed_fees: +bit_com_perpetual_testnet_taker_fixed_fees: +whitebit_percent_fee_token: +whitebit_maker_percent_fee: +whitebit_taker_percent_fee: +whitebit_buy_percent_fee_deducted_from_returns: +whitebit_maker_fixed_fees: +whitebit_taker_fixed_fees: +bitmex_percent_fee_token: +bitmex_maker_percent_fee: +bitmex_taker_percent_fee: +bitmex_buy_percent_fee_deducted_from_returns: +bitmex_maker_fixed_fees: +bitmex_taker_fixed_fees: +bitmex_testnet_percent_fee_token: +bitmex_testnet_maker_percent_fee: +bitmex_testnet_taker_percent_fee: +bitmex_testnet_buy_percent_fee_deducted_from_returns: +bitmex_testnet_maker_fixed_fees: +bitmex_testnet_taker_fixed_fees: +ciex_percent_fee_token: +ciex_maker_percent_fee: +ciex_taker_percent_fee: +ciex_buy_percent_fee_deducted_from_returns: +ciex_maker_fixed_fees: +ciex_taker_fixed_fees: +foxbit_percent_fee_token: +foxbit_maker_percent_fee: +foxbit_taker_percent_fee: +foxbit_buy_percent_fee_deducted_from_returns: +foxbit_maker_fixed_fees: +foxbit_taker_fixed_fees: +lbank_percent_fee_token: +lbank_maker_percent_fee: +lbank_taker_percent_fee: +lbank_buy_percent_fee_deducted_from_returns: +lbank_maker_fixed_fees: +lbank_taker_fixed_fees: +bybit_percent_fee_token: +bybit_maker_percent_fee: +bybit_taker_percent_fee: +bybit_buy_percent_fee_deducted_from_returns: +bybit_maker_fixed_fees: +bybit_taker_fixed_fees: +bybit_testnet_percent_fee_token: +bybit_testnet_maker_percent_fee: +bybit_testnet_taker_percent_fee: +bybit_testnet_buy_percent_fee_deducted_from_returns: +bybit_testnet_maker_fixed_fees: +bybit_testnet_taker_fixed_fees: +hotbit_percent_fee_token: +hotbit_maker_percent_fee: +hotbit_taker_percent_fee: +hotbit_buy_percent_fee_deducted_from_returns: +hotbit_maker_fixed_fees: +hotbit_taker_fixed_fees: +btc_markets_maker_fixed_fees: +btc_markets_taker_fixed_fees: diff --git a/hummingbot_files/conf/hummingbot_logs.yml b/hummingbot_files/conf/hummingbot_logs.yml new file mode 100755 index 0000000..8e65271 --- /dev/null +++ b/hummingbot_files/conf/hummingbot_logs.yml @@ -0,0 +1,83 @@ +--- +version: 1 +template_version: 12 + +formatters: + simple: + format: "%(asctime)s - %(process)d - %(name)s - %(levelname)s - %(message)s" + +handlers: + console: + class: hummingbot.logger.cli_handler.CLIHandler + level: DEBUG + formatter: simple + stream: ext://sys.stdout + console_warning: + class: hummingbot.logger.cli_handler.CLIHandler + level: WARNING + formatter: simple + stream: ext://sys.stdout + console_info: + class: hummingbot.logger.cli_handler.CLIHandler + level: INFO + formatter: simple + stream: ext://sys.stdout + file_handler: + class: logging.handlers.TimedRotatingFileHandler + level: DEBUG + formatter: simple + filename: $PROJECT_DIR/logs/logs_$STRATEGY_FILE_PATH.log + encoding: utf8 + when: "D" + interval: 1 + backupCount: 7 + "null": + class: logging.NullHandler + level: DEBUG + +loggers: + hummingbot.core.utils.eth_gas_station_lookup: + level: NETWORK + propagate: false + handlers: [console, file_handler] + mqtt: true + hummingbot.logger.log_server_client: + level: WARNING + propagate: false + handlers: [console, file_handler] + mqtt: true + hummingbot.logger.reporting_proxy_handler: + level: WARNING + propagate: false + handlers: [console, file_handler] + mqtt: true + hummingbot.strategy: + level: NETWORK + propagate: false + handlers: [console, file_handler] + mqtt: true + hummingbot.connector: + level: NETWORK + propagate: false + handlers: [console, file_handler] + mqtt: true + hummingbot.client: + level: NETWORK + propagate: false + handlers: [console, file_handler] + mqtt: true + hummingbot.core.event.event_reporter: + level: EVENT_LOG + propagate: false + handlers: [file_handler] + mqtt: false + conf: + level: NETWORK + handlers: ["null"] + propagate: false + mqtt: false + +root: + level: INFO + handlers: [console, file_handler] + mqtt: true From 2e1164ea3230dc393ef5d164be79a239fd4fee00 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 19:46:23 -0300 Subject: [PATCH 07/14] (feat) add compose file and yml to run hummingbot candles downlaoder --- .../compose_files/data-downloader-compose.yml | 22 +++++++++++++++++++ .../data_downloader_config.yml | 4 ++++ 2 files changed, 26 insertions(+) create mode 100644 hummingbot_files/compose_files/data-downloader-compose.yml create mode 100644 hummingbot_files/scripts_configs/data_downloader_config.yml diff --git a/hummingbot_files/compose_files/data-downloader-compose.yml b/hummingbot_files/compose_files/data-downloader-compose.yml new file mode 100644 index 0000000..2ce3372 --- /dev/null +++ b/hummingbot_files/compose_files/data-downloader-compose.yml @@ -0,0 +1,22 @@ +version: "3.9" +services: + bot: + container_name: data_downloader + image: dardonacci/hummingbot:dev + volumes: + - "../../data/candles:/home/hummingbot/data" + - "../conf:/home/hummingbot/conf" + - "../conf/connectors:/home/hummingbot/conf/connectors" + environment: + - CONFIG_PASSWORD=a + - CONFIG_FILE_NAME=download_candles.py + env_file: + - ../scripts_configs/data_downloader_config.yml + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: 5 + tty: true + stdin_open: true + network_mode: host diff --git a/hummingbot_files/scripts_configs/data_downloader_config.yml b/hummingbot_files/scripts_configs/data_downloader_config.yml new file mode 100644 index 0000000..d774177 --- /dev/null +++ b/hummingbot_files/scripts_configs/data_downloader_config.yml @@ -0,0 +1,4 @@ +DAYS_TO_DOWNLOAD: 3 +EXCHANGE: binance_perpetual +INTERVALS: 1m,3m,1h +TRADING_PAIRS: BTC-USDT,ETH-USDT From 0fbd83db68d9b87b9407272fc8f6bf30bd9f235a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 1 Jun 2023 19:46:38 -0300 Subject: [PATCH 08/14] (feat) create candles downloader page --- pages/6_πŸ—‚_Candles_Downloader.py | 75 +++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 pages/6_πŸ—‚_Candles_Downloader.py diff --git a/pages/6_πŸ—‚_Candles_Downloader.py b/pages/6_πŸ—‚_Candles_Downloader.py new file mode 100644 index 0000000..cf514a0 --- /dev/null +++ b/pages/6_πŸ—‚_Candles_Downloader.py @@ -0,0 +1,75 @@ +import time +from subprocess import CalledProcessError + +import streamlit as st + +import constants +from utils import os_utils +from utils.docker_manager import DockerManager + +st.set_page_config( + page_title="Candles Downloader", + page_icon=":bar_chart:", + layout="wide", +) +docker_manager = DockerManager() +st.write(f"# πŸ—‚οΈ Candles Downloader") +st.write("---") + +c1, c2, c3 = st.columns([2, 2, 0.5]) +with c1: + exchange = st.selectbox("Exchange", ["binance_perpetual", "binance"], index=0) + # TODO: Add more trading pairs querying the exchange API + trading_pairs = st.text_input("Trading Pairs (separated with commas)", value="BTC-USDT,ETH-USDT") +with c2: + intervals = st.multiselect("Intervals", options=["1m", "3m", "5m", "15m", "1h", "4h", "1d"], default=["1m", "3m", "1h"]) + days_to_download = st.number_input("Days to Download", value=30, min_value=1, max_value=365, step=1) +with c3: + get_data_button = st.button("Download Candles!") + clean_container_folder_button = st.button("Clean Candles Folder") + + +if clean_container_folder_button: + st.warning("Cleaning Candles Data folder...", icon="⚠️") + st.write("---") + os_utils.remove_files_from_directory(constants.CANDLES_DATA_PATH) + st.write("### Container folder cleaned.") + st.write("---") + +if get_data_button: + candles_container_config = { + "EXCHANGE": exchange, + "TRADING_PAIRS": trading_pairs, + "INTERVALS": ",".join(intervals), + "DAYS_TO_DOWNLOAD": days_to_download, + } + time.sleep(0.5) + docker_manager.create_download_candles_container(candles_container_config) + st.write("---") + +st.write("---") +st.write("## βš™οΈContainers Management") +try: + + c1, c2 = st.columns([0.85, 0.15]) + with c1: + st.write("Active Containers:") + st.info(docker_manager.get_active_containers()) + + st.write("Exited Containers:") + st.warning(docker_manager.get_exited_containers()) + with c2: + stop_containers_button = st.button("Stop Containers") + if stop_containers_button: + docker_manager.stop_active_containers() + + clean_exited_containers_button = st.button("Clean Containers") + if clean_exited_containers_button: + docker_manager.clean_exited_containers() + + refresh_button = st.button("Refresh") + if refresh_button: + pass +except CalledProcessError as error: + st.write("### Docker is not running. Please start docker in your machine.") + From 8e0f75ea6c30eb613512aee23d509488e32f807b Mon Sep 17 00:00:00 2001 From: Michael Feng Date: Sat, 3 Jun 2023 12:10:47 -0700 Subject: [PATCH 09/14] (doc) add update and candles downloader to README --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 850a793..8157174 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,14 @@ These files are located in the `/data` folder in Hummingbot, and are named ` Date: Sun, 4 Jun 2023 15:10:36 +0200 Subject: [PATCH 10/14] fix: empty database causing the usage of undefined dashboard parameters --- pages/2_πŸš€_Strategy_Performance.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pages/2_πŸš€_Strategy_Performance.py b/pages/2_πŸš€_Strategy_Performance.py index d466d10..2b334af 100644 --- a/pages/2_πŸš€_Strategy_Performance.py +++ b/pages/2_πŸš€_Strategy_Performance.py @@ -53,18 +53,22 @@ with st.container(): st.warning("No databases available to analyze. Please run a backtesting first.") else: db_manager = get_database(selected_db_name) + config_files = db_manager.get_config_files() + if config_files == []: + with col1: + st.warning('No trades have been recorded in the selected database') with col2: - selected_config_file = st.selectbox("Select a config file to analyze:", db_manager.get_config_files()) + selected_config_file = st.selectbox("Select a config file to analyze:", config_files) if selected_config_file is not None: exchanges_trading_pairs = db_manager.get_exchanges_trading_pairs_by_config_file(selected_config_file) strategy_data = db_manager.get_strategy_data(selected_config_file) - + with st.container(): col1, col2, col3 = st.columns(3) with col1: - selected_exchange = st.selectbox("Select an exchange:", list(exchanges_trading_pairs.keys())) + selected_exchange = st.selectbox("Select an exchange:", [] if selected_config_file is None else list(exchanges_trading_pairs.keys())) with col2: - selected_trading_pair = st.selectbox("Select a trading pair:", exchanges_trading_pairs[selected_exchange]) + selected_trading_pair = st.selectbox("Select a trading pair:", [] if selected_config_file is None else exchanges_trading_pairs[selected_exchange]) with col3: interval = st.selectbox("Candles Interval:", intervals.keys(), index=0) From 34ef1090cd1aee9ea757eee4cabff4bcc9c06ccd Mon Sep 17 00:00:00 2001 From: devamin Date: Sun, 4 Jun 2023 15:11:52 +0200 Subject: [PATCH 11/14] feat: vscode debuging settings --- .vscode/launch.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..c58a7d5 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: streamlit", + "type": "python", + "request": "launch", + "module": "streamlit", + "args": [ + "run", + "main.py" + ], + "justMyCode": true + } + ] +} \ No newline at end of file From e3fe7854402e473759c9091a2dd6212d19e89aa2 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 9 Jun 2023 16:04:58 +0100 Subject: [PATCH 12/14] (feat) change base compose image --- hummingbot_files/compose_files/data-downloader-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot_files/compose_files/data-downloader-compose.yml b/hummingbot_files/compose_files/data-downloader-compose.yml index 2ce3372..0d5d65c 100644 --- a/hummingbot_files/compose_files/data-downloader-compose.yml +++ b/hummingbot_files/compose_files/data-downloader-compose.yml @@ -2,7 +2,7 @@ version: "3.9" services: bot: container_name: data_downloader - image: dardonacci/hummingbot:dev + image: hummingbot/hummingbot:development volumes: - "../../data/candles:/home/hummingbot/data" - "../conf:/home/hummingbot/conf" From e315cc6abae574d62addec0f92a07ff9bbed1c10 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 9 Jun 2023 16:05:43 +0100 Subject: [PATCH 13/14] (feat) add info of where the data will be stored --- pages/6_πŸ—‚_Candles_Downloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/6_πŸ—‚_Candles_Downloader.py b/pages/6_πŸ—‚_Candles_Downloader.py index cf514a0..78e1679 100644 --- a/pages/6_πŸ—‚_Candles_Downloader.py +++ b/pages/6_πŸ—‚_Candles_Downloader.py @@ -19,7 +19,6 @@ st.write("---") c1, c2, c3 = st.columns([2, 2, 0.5]) with c1: exchange = st.selectbox("Exchange", ["binance_perpetual", "binance"], index=0) - # TODO: Add more trading pairs querying the exchange API trading_pairs = st.text_input("Trading Pairs (separated with commas)", value="BTC-USDT,ETH-USDT") with c2: intervals = st.multiselect("Intervals", options=["1m", "3m", "5m", "15m", "1h", "4h", "1d"], default=["1m", "3m", "1h"]) @@ -28,7 +27,6 @@ with c3: get_data_button = st.button("Download Candles!") clean_container_folder_button = st.button("Clean Candles Folder") - if clean_container_folder_button: st.warning("Cleaning Candles Data folder...", icon="⚠️") st.write("---") @@ -45,6 +43,8 @@ if get_data_button: } time.sleep(0.5) docker_manager.create_download_candles_container(candles_container_config) + st.info("Downloading candles with a Docker container in the background. " + "When this process is ready you will see the candles inside data/candles", icon="πŸ•“") st.write("---") st.write("---") From 7b581d74d314c15a2996ebd44ddefaa433c0938f Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 9 Jun 2023 16:05:58 +0100 Subject: [PATCH 14/14] (feat) revert to dardonacci to have multi-arch --- hummingbot_files/compose_files/data-downloader-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot_files/compose_files/data-downloader-compose.yml b/hummingbot_files/compose_files/data-downloader-compose.yml index 0d5d65c..f822005 100644 --- a/hummingbot_files/compose_files/data-downloader-compose.yml +++ b/hummingbot_files/compose_files/data-downloader-compose.yml @@ -2,7 +2,7 @@ version: "3.9" services: bot: container_name: data_downloader - image: hummingbot/hummingbot:development + image: dardonacci/hummingbot:development volumes: - "../../data/candles:/home/hummingbot/data" - "../conf:/home/hummingbot/conf"