# --- Do not remove these libs --- import logging import time import datetime import pandas as pd import freqtrade.vendor.qtpylib.indicators as qtpylib import talib.abstract as ta import numpy as np import re from freqtrade.strategy import IStrategy, merge_informative_pair, DecimalParameter, IntParameter from pandas import DataFrame, Series from technical.indicators import zema # -------------------------------- """ The Matrix - by @werkkrew Denial is the most predictable of all human responses. VERSION = 0.0.1.4 NOTE NOTE NOTE PLEASE READ: - This is a proof-of-concept for using the pearson correlation coefficient in the buy signal. - Uses zemaoff because it's simple with few indicators and was easier to get up for a POC. - Technically you can use the buy from any strategy within this framework but some are certainly going to work better than others. - If you want to experiment with other indicators keep in mind that this strategy will buy coin X based on the signals from some # of other coins, and not itself. General Premise: 1) Dynamically create a correlation matrix such as the one at: https://cryptowat.ch/correlations based on current whitelist 2) Look to buy when a large # of highly correlated pairs signal a buy based on TA 3) ... 4) Profit? - You can hyperopt it, but you'll have to be careful about memory, experiment. If you want to try: - Use a reasonably low -j - Use a small-ish timerange (<30 days) - Use a reasonable pairlist (<40 pairs) TODO: - Experiement with identifying "laggy" coins and buy when correlated pairs signal a buy but coin itself does not. - Same on the sell side """ logger = logging.getLogger(__name__) class Matrix(IStrategy): corr_timeframe = '1h' # don't change this for now, must of the code is hard-coded around this value at this time. corr_period = 24 # of above timeframe candles minimal_roi = { "0": 0.081, "10": 0.038, "20": 0.012, "120": 0 } buy_params = { "buy_offset": 0.819, "buy_min_corr_coef": 0.867, "buy_min_corr_pair": 19, "buy_zema_len": 37 } sell_params = { "sell_offset": 0.845, "sell_min_corr_coef": 0.867, "sell_min_corr_pair": 19, "sell_zema_len": 63, } stoploss = -0.3 timeframe = '5m' use_sell_signal = True sell_profit_only = True process_only_new_candles = True ignore_roi_if_buy_signal = True # Minimum correlation coefficient to use coin as a correlated signal buy_min_corr_coef = DecimalParameter(0.50, 0.95, default=0.85, space='buy', optimize=True, load=True) sell_min_corr_coef = DecimalParameter(0.50, 0.95, default=0.85, space='sell', optimize=True, load=True) # Minimum number of correlated buy signals which need to pass to buy or sell buy_min_corr_pairs = IntParameter(3, 20, default=5, space='buy', optimize=True, load=True) sell_min_corr_pairs = IntParameter(3, 20, default=5, space='sell', optimize=True, load=True) buy_offset = DecimalParameter(0.80, 1.20, default=1.004, space='buy', optimize=True, load=True) buy_zema_len = IntParameter(30, 90, default=72, space='buy', optimize=True, load=True) sell_offset = DecimalParameter(0.80, 1.20, default=0.964, space='sell', optimize=True, load=True) sell_zema_len = IntParameter(30, 90, default=51, space='sell', optimize=True, load=True) startup_candle_count = max(corr_period, buy_zema_len.value) # Custom variable storage (don't fuck with these) meta_matrix = DataFrame() meta_indicators = DataFrame() # Cache the whitelist so we can detect changes whitelist = {} metaframe_status = {} metaframe_status['indicators_built'] = False metaframe_status['matrix_built'] = False metaframe_status['when'] = datetime.datetime.now() """ Reset the status of meta indicators at the start of each loop Reset the status of the correlations near the top of the hour or on whitelist content change Don't edit in here. """ def bot_loop_start(self, **kwargs) -> None: self.metaframe_status['indicators_built'] = False rebuild_matrix = False mins_since = (datetime.datetime.now() - self.metaframe_status['when']).total_seconds() // 60 mins_after = int(time.strftime('%M')) # Fuzzy logic here because we don't know how long the bot loop will take so we provide a window of 5 minutes. if (mins_after > 0 and mins_after <= 5) and mins_since > 5: logger.info(f"New 1h Candle - Resetting correlation matrix build status...") rebuild_matrix = True # If the contents of the matrix are not aligned with the current whitelist, rebuild it. if self.whitelist != self.dp.current_whitelist(): logger.info(f"Whitelist Changed - Resetting correlation matrix build status...") rebuild_matrix = True if rebuild_matrix: self.metaframe_status['matrix_built'] = False self.metaframe_status['when'] = datetime.datetime.now() """ Build a "metaframe" including the informative timeframe close price for all pairs in whitelist as well as build a multiindex pairwise correlation table for all of the data. Don't edit in here. """ def build_matrix(self): if not self.metaframe_status['matrix_built']: # Reset the dataframe back to empty self.meta_matrix = DataFrame() logger.info("Building Metaframe...") tic = time.perf_counter() pairs = self.dp.current_whitelist() # Cache the whitelist at the time matrix was built self.whitelist = pairs metaframe = DataFrame() # Iterate through all pairs in our whitelist and add the close price to the metaframe on matching date for pair in pairs: coin, _ = pair.split('/') # Get the close price for the informative timeframe and name the column for the coin corr = self.dp.get_pair_dataframe(pair=pair, timeframe=self.corr_timeframe) corr[coin] = corr['close'] # Drop the columns we don't need corr = corr.drop(columns=['open', 'high', 'low', 'close', 'volume']) # Set the date index of the metaframe once if not 'date' in metaframe: metaframe['date'] = corr['date'] metaframe.set_index('date') # Merge it in on the date key metaframe = pd.merge(metaframe, corr, on='date', how='left') toc = time.perf_counter() logger.info(f"Metaframe build took {toc - tic:0.4f} seconds...") # Build the correlation matrix logger.info("Processing Correlations...") tic = time.perf_counter() self.meta_matrix = metaframe.set_index('date').rolling(self.corr_period).corr(pairwise=True) toc = time.perf_counter() logger.info(f"Correlation processing took {toc - tic:0.4f} seconds...") # Mark metaframe as having been built so subsequent calls to populate_indicators don't do it again self.metaframe_status['matrix_built'] = True self.metaframe_status['when'] = datetime.datetime.now() """ Similar to the metaframe, build a meta indicators dataframe so we only calculate indicators for each coin once per loop iteration to attempt to save on processing. Don't edit in here. """ def build_meta_indicators(self): if not self.metaframe_status['indicators_built']: # Reset the dataframe back to empty self.meta_indicators = DataFrame() logger.info("Calculating Indicators...") tic = time.perf_counter() pairs = self.dp.current_whitelist() # Iterate through every pair in the whitelist for pair in pairs: coin, _ = pair.split('/') df = self.dp.get_pair_dataframe(pair=pair, timeframe=self.timeframe) # The indicator stuff is in a different method to make strategy development easier df = self.populate_meta_indicators(df, coin) # Drop the columns we don't need df = df.drop(columns=['open', 'high', 'low', 'close', 'volume']) # Set the date index of the metaframe once if not 'date' in self.meta_indicators: self.meta_indicators['date'] = df['date'] self.meta_indicators.set_index('date') # Merge it in on the date key self.meta_indicators = pd.merge(self.meta_indicators, df, on='date', how='left') self.metaframe_status['indicators_built'] = True toc = time.perf_counter() logger.info(f"Indicator calcuation took {toc - tic:0.4f} seconds...") """ Merge the correlations for the current coin from the metaframe into the coins dataframe Don't edit in here. """ def merge_correlations(self, dataframe: DataFrame, pair: str): coin, _ = pair.split('/') # Get informative timeframe data for current pair informative = self.dp.get_pair_dataframe(pair=pair, timeframe=self.corr_timeframe) informative = informative.drop(columns=['open', 'high', 'low', 'close', 'volume']) # Filter the correlation matrix by the current coin and remove columns we don't need coin_correlation = self.meta_matrix[self.meta_matrix.index.isin([coin], level=1)] coin_correlation = coin_correlation.droplevel(1) coin_correlation = coin_correlation.drop(columns=[coin]) # Merge the filtered matrix into informative on date informative = pd.merge(informative, coin_correlation, on='date', how='left') # Merge back into primary dataframe using standard process and cross fingers? dataframe = merge_informative_pair(dataframe, informative, self.timeframe, self.corr_timeframe, ffill=True) return dataframe def informative_pairs(self): # add all whitelisted pairs on informative timeframe pairs = self.dp.current_whitelist() informative_pairs = [(pair, self.corr_timeframe) for pair in pairs] return informative_pairs """ Any indicators used in the buy signal should go in here, this is calculated for every coin so using {prefix} as a prefix in the column name is required Ok to edit in here for strategy specific indicators. """ def populate_meta_indicators(self, dataframe: DataFrame, coin: str) -> DataFrame: # Set a unique prefix to make splitting the multiindex columns easier later prefix = f'{coin}@' # Calculate indicators for the coin for length in self.buy_zema_len.range: dataframe[f'{prefix}zema_{length}'] = zema(dataframe, period=length) for length in self.sell_zema_len.range: if f'{prefix}zema_{length}' not in dataframe: dataframe[f'{prefix}zema_{length}'] = zema(dataframe, period=length) dataframe[f'{prefix}close'] = dataframe['close'] return dataframe """ Don't edit in here, the indicators don't go here! See above! """ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Build the correlation matrix self.build_matrix() # Build the meta indicators (calculate the indicators for all whitelist pairs once) self.build_meta_indicators() # Filter the correlation metaframe by this specific pair and merge those correlations into the main dataframe dataframe = self.merge_correlations(dataframe, metadata['pair']) # Rename the COIN_1h columns to COIN@coef for consistency and splitting later dataframe = dataframe.rename(columns=lambda x: re.sub("_1h", "@coef", x)) # Merge all of the indicators into this coins dataframe dataframe = pd.merge(dataframe, self.meta_indicators, on='date', how='left') return dataframe """ Only edit the one section in here, don't fuck with the rest. """ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Create a new dataframe without the columns we don't need as a multi-index per coin # Investigate: https://towardsdatascience.com/how-to-use-the-split-apply-combine-strategy-in-pandas-groupby-29e0eb44b62e buy_df = dataframe.drop(columns=['date', 'open', 'high', 'low', 'close', 'volume', 'date@coef']) buy_df.columns = buy_df.columns.str.split('@', expand=True) buy_df = buy_df.stack(level=0) # Primary buy conditions go here # THIS IS WHERE YOU EDIT THE BUY SIGNAL, NOWHERE ELSE! dataframe['buy_count'] = ( (buy_df['coef'].gt(self.buy_min_corr_coef.value)) & (buy_df['close'].lt(buy_df[f'zema_{self.buy_zema_len.value}'] * self.buy_offset.value)) ).sum(level=0) # END SUPER NORMAL BUY SIGNALS # Evaluate if we should buy based on if enough of the above buys happened # Don't edit this part, probably. dataframe.loc[ ( (dataframe['buy_count'] >= self.buy_min_corr_pairs.value) & (dataframe['volume'] > 0) ),'buy'] = 1 return dataframe """ Ok to edit this part. """ def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Create a new dataframe without the columns we don't need as a multi-index per coin sell_df = dataframe.drop(columns=['date', 'open', 'high', 'low', 'close', 'volume', 'date@coef']) sell_df.columns = sell_df.columns.str.split('@', expand=True) sell_df = sell_df.stack(level=0) # Primary sell conditions go here # THIS IS WHERE YOU EDIT THE SELL SIGNAL, NOWHERE ELSE! dataframe['sell_count'] = ( (sell_df['coef'].gt(self.sell_min_corr_coef.value)) & (sell_df['close'].gt(sell_df[f'zema_{self.sell_zema_len.value}'] * self.sell_offset.value)) ).sum(level=0) # END SUPER NORMAL SELL SIGNALS # Evaluate if we should sell based on if enough of the above sells happened # Don't edit this part, probably. dataframe.loc[ ( (dataframe['sell_count'] >= self.sell_min_corr_pairs.value) & (dataframe['volume'] > 0) ),'sell'] = 1 return dataframe # https://www.youtube.com/watch?v=Mb4fzjsuMYI