mirror of
https://github.com/aljazceru/hummingbot-dashboard.git
synced 2025-12-27 18:24:22 +01:00
204 lines
9.7 KiB
Python
204 lines
9.7 KiB
Python
import os
|
|
import streamlit as st
|
|
|
|
import pandas as pd
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from utils.data_manipulation import StrategyData
|
|
|
|
|
|
class DatabaseManager:
|
|
def __init__(self, db_name: str, executors_path: str = "data"):
|
|
self.db_name = db_name
|
|
# TODO: Create db path for all types of db
|
|
self.db_path = f'sqlite:///{os.path.join(db_name)}'
|
|
self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False})
|
|
self.session_maker = sessionmaker(bind=self.engine)
|
|
|
|
def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None):
|
|
def load_data(table_loader):
|
|
try:
|
|
return table_loader()
|
|
except Exception as e:
|
|
return None # Return None to indicate failure
|
|
|
|
# Use load_data to load tables
|
|
orders = load_data(self.get_orders)
|
|
trade_fills = load_data(self.get_trade_fills)
|
|
order_status = load_data(self.get_order_status)
|
|
market_data = load_data(self.get_market_data)
|
|
position_executor = load_data(self.get_position_executor_data)
|
|
|
|
strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor)
|
|
return strategy_data
|
|
|
|
@staticmethod
|
|
def _get_table_status(table_loader):
|
|
try:
|
|
data = table_loader()
|
|
return "Correct" if len(data) > 0 else f"Error - No records matched"
|
|
except Exception as e:
|
|
return f"Error - {str(e)}"
|
|
|
|
@property
|
|
def status(self):
|
|
status = {"db_name": self.db_name,
|
|
"trade_fill": self._get_table_status(self.get_trade_fills),
|
|
"orders": self._get_table_status(self.get_orders),
|
|
"order_status": self._get_table_status(self.get_order_status),
|
|
"market_data": self._get_table_status(self.get_market_data),
|
|
"position_executor": self._get_table_status(self.get_position_executor_data),
|
|
}
|
|
return status
|
|
|
|
@property
|
|
def config_files(self):
|
|
return self.get_config_files()
|
|
|
|
@property
|
|
def configs(self):
|
|
return {config_file: self.get_exchanges_trading_pairs_by_config_file(config_file) for config_file in self.config_files}
|
|
|
|
def get_config_files(self):
|
|
with self.session_maker() as session:
|
|
query = 'SELECT DISTINCT config_file_path FROM TradeFill'
|
|
config_files = pd.read_sql_query(text(query), session.connection())
|
|
return config_files['config_file_path'].tolist()
|
|
|
|
def get_exchanges_trading_pairs_by_config_file(self, config_file_path):
|
|
with self.session_maker() as session:
|
|
query = f"SELECT DISTINCT market, symbol FROM TradeFill WHERE config_file_path = '{config_file_path}'"
|
|
exchanges_trading_pairs = pd.read_sql_query(text(query), session.connection())
|
|
exchanges_trading_pairs["market"] = exchanges_trading_pairs["market"].apply(
|
|
lambda x: x.lower().replace("_papertrade", ""))
|
|
exchanges_trading_pairs = exchanges_trading_pairs.groupby("market")["symbol"].apply(list).to_dict()
|
|
return exchanges_trading_pairs
|
|
|
|
@staticmethod
|
|
def _get_orders_query(config_file_path=None, start_date=None, end_date=None):
|
|
query = "SELECT * FROM 'Order'"
|
|
conditions = []
|
|
if config_file_path:
|
|
conditions.append(f"config_file_path = '{config_file_path}'")
|
|
if start_date:
|
|
conditions.append(f"created_at >= '{start_date}'")
|
|
if end_date:
|
|
conditions.append(f"created_at <= '{end_date}'")
|
|
if conditions:
|
|
query += f" WHERE {' AND '.join(conditions)}"
|
|
return query
|
|
|
|
@staticmethod
|
|
def _get_order_status_query(order_ids=None, start_date=None, end_date=None):
|
|
query = "SELECT * FROM OrderStatus"
|
|
conditions = []
|
|
if order_ids:
|
|
order_ids_string = ",".join(f"'{order_id}'" for order_id in order_ids)
|
|
conditions.append(f"order_id IN ({order_ids_string})")
|
|
if start_date:
|
|
conditions.append(f"created_at >= '{start_date}'")
|
|
if end_date:
|
|
conditions.append(f"created_at <= '{end_date}'")
|
|
if conditions:
|
|
query += f" WHERE {' AND '.join(conditions)}"
|
|
return query
|
|
|
|
@staticmethod
|
|
def _get_trade_fills_query(config_file_path=None, start_date=None, end_date=None):
|
|
query = "SELECT * FROM TradeFill"
|
|
conditions = []
|
|
if config_file_path:
|
|
conditions.append(f"config_file_path = '{config_file_path}'")
|
|
if start_date:
|
|
conditions.append(f"created_at >= '{start_date}'")
|
|
if end_date:
|
|
conditions.append(f"created_at <= '{end_date}'")
|
|
if conditions:
|
|
query += f" WHERE {' AND '.join(conditions)}"
|
|
return query
|
|
|
|
@staticmethod
|
|
def _get_market_data_query(start_date=None, end_date=None):
|
|
query = "SELECT * FROM MarketData"
|
|
conditions = []
|
|
if start_date:
|
|
conditions.append(f"timestamp >= '{start_date * 1e6}'")
|
|
if end_date:
|
|
conditions.append(f"timestamp <= '{end_date * 1e6}'")
|
|
if conditions:
|
|
query += f" WHERE {' AND '.join(conditions)}"
|
|
return query
|
|
|
|
@staticmethod
|
|
def _get_position_executor_query(start_date=None, end_date=None):
|
|
query = "SELECT * FROM PositionExecutors"
|
|
conditions = []
|
|
if start_date:
|
|
conditions.append(f"timestamp >= '{start_date}'")
|
|
if end_date:
|
|
conditions.append(f"timestamp <= '{end_date}'")
|
|
if conditions:
|
|
query += f" WHERE {' AND '.join(conditions)}"
|
|
return query
|
|
|
|
def get_orders(self, config_file_path=None, start_date=None, end_date=None):
|
|
with self.session_maker() as session:
|
|
query = self._get_orders_query(config_file_path, start_date, end_date)
|
|
orders = pd.read_sql_query(text(query), session.connection())
|
|
orders["market"] = orders["market"]
|
|
orders["amount"] = orders["amount"] / 1e6
|
|
orders["price"] = orders["price"] / 1e6
|
|
orders['creation_timestamp'] = pd.to_datetime(orders['creation_timestamp'], unit="ms")
|
|
orders['last_update_timestamp'] = pd.to_datetime(orders['last_update_timestamp'], unit="ms")
|
|
return orders
|
|
|
|
def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None):
|
|
groupers = ["config_file_path", "market", "symbol"]
|
|
float_cols = ["amount", "price", "trade_fee_in_quote"]
|
|
with self.session_maker() as session:
|
|
query = self._get_trade_fills_query(config_file_path, start_date, end_date)
|
|
trade_fills = pd.read_sql_query(text(query), session.connection())
|
|
trade_fills[float_cols] = trade_fills[float_cols] / 1e6
|
|
trade_fills["cum_fees_in_quote"] = trade_fills.groupby(groupers)["trade_fee_in_quote"].cumsum()
|
|
trade_fills["net_amount"] = trade_fills['amount'] * trade_fills['trade_type'].apply(lambda x: 1 if x == 'BUY' else -1)
|
|
trade_fills["net_amount_quote"] = trade_fills['net_amount'] * trade_fills['price']
|
|
trade_fills["cum_net_amount"] = trade_fills.groupby(groupers)["net_amount"].cumsum()
|
|
trade_fills["unrealized_trade_pnl"] = -1 * trade_fills.groupby(groupers)["net_amount_quote"].cumsum()
|
|
trade_fills["inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"]
|
|
trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"]
|
|
trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"]
|
|
trade_fills["realized_pnl"] = trade_fills.groupby(groupers)["net_realized_pnl"].diff()
|
|
trade_fills["gross_pnl"] = trade_fills.groupby(groupers)["realized_trade_pnl"].diff()
|
|
trade_fills["trade_fee"] = trade_fills.groupby(groupers)["cum_fees_in_quote"].diff()
|
|
trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms")
|
|
trade_fills["market"] = trade_fills["market"]
|
|
trade_fills["quote_volume"] = trade_fills["price"] * trade_fills["amount"]
|
|
return trade_fills
|
|
|
|
def get_order_status(self, order_ids=None, start_date=None, end_date=None):
|
|
with self.session_maker() as session:
|
|
query = self._get_order_status_query(order_ids, start_date, end_date)
|
|
order_status = pd.read_sql_query(text(query), session.connection())
|
|
return order_status
|
|
|
|
def get_market_data(self, start_date=None, end_date=None):
|
|
with self.session_maker() as session:
|
|
query = self._get_market_data_query(start_date, end_date)
|
|
market_data = pd.read_sql_query(text(query), session.connection())
|
|
market_data["timestamp"] = pd.to_datetime(market_data["timestamp"] / 1e6, unit="ms")
|
|
market_data.set_index("timestamp", inplace=True)
|
|
market_data["mid_price"] = market_data["mid_price"] / 1e6
|
|
market_data["best_bid"] = market_data["best_bid"] / 1e6
|
|
market_data["best_ask"] = market_data["best_ask"] / 1e6
|
|
return market_data
|
|
|
|
def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataFrame:
|
|
with self.session_maker() as session:
|
|
query = self._get_position_executor_query(start_date, end_date)
|
|
position_executor = pd.read_sql_query(text(query), session.connection())
|
|
position_executor.set_index("timestamp", inplace=True)
|
|
position_executor["datetime"] = pd.to_datetime(position_executor.index, unit="s")
|
|
position_executor["level"] = position_executor["order_level"].apply(lambda x: x.split("_")[1])
|
|
return position_executor
|