# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement # flake8: noqa: F401 # --- Do not remove these libs --- import numpy as np # noqa import pandas as pd # noqa from pandas import DataFrame # noqa from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, IStrategy, IntParameter) # -------------------------------- # Add your lib to import here import talib.abstract as ta import pandas_ta as pta import freqtrade.vendor.qtpylib.indicators as qtpylib from DQN import Agent from utils import * import logging logger = logging.getLogger(__name__) class TrainDeepQStrategy(IStrategy): # Strategy interface version - allow new iterations of the strategy interface. # Check the documentation or the Sample strategy to get the latest version. INTERFACE_VERSION = 3 # Optimal timeframe for the strategy. timeframe = '5m' num_episode = 5 window_size = 10+3 agent = Agent(window_size, False) #Training action_dict = {0: 'Hold', 1: 'Buy', 2: 'Sell'} # Can this strategy go short? can_short: bool = False # Minimal ROI designed for the strategy. # This attribute will be overridden if the config file contains "minimal_roi". minimal_roi = { "60": 0.01, "30": 0.02, "0": 0.04 } # Optimal stoploss designed for the strategy. # This attribute will be overridden if the config file contains "stoploss". stoploss = -0.10 # Trailing stoploss trailing_stop = False # trailing_only_offset_is_reached = False # trailing_stop_positive = 0.01 # trailing_stop_positive_offset = 0.0 # Disabled / not configured # Run "populate_indicators()" only for new candle. process_only_new_candles = True # These values can be overridden in the config. use_exit_signal = True exit_profit_only = False ignore_roi_if_entry_signal = False # Number of candles the strategy requires before producing valid signals startup_candle_count: int = 30 # Optional order type mapping. order_types = { 'entry': 'limit', 'exit': 'limit', 'stoploss': 'market', 'stoploss_on_exchange': False } # Optional order time in force. order_time_in_force = { 'entry': 'gtc', 'exit': 'gtc' } @property def plot_config(self): return { # Main plot indicators (Moving averages, ...) 'main_plot': { }, 'subplots': { } } def informative_pairs(self): return [] def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Momentum Indicators # ------------------------------------ # ADX dataframe['adx'] = ta.ADX(dataframe) # # Plus Directional Indicator / Movement # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) # dataframe['plus_di'] = ta.PLUS_DI(dataframe) # # Minus Directional Indicator / Movement # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) # dataframe['minus_di'] = ta.MINUS_DI(dataframe) # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) # # Keltner Channel # keltner = qtpylib.keltner_channel(dataframe) # dataframe["kc_upperband"] = keltner["upper"] # dataframe["kc_lowerband"] = keltner["lower"] # dataframe["kc_middleband"] = keltner["mid"] # dataframe["kc_percent"] = ( # (dataframe["close"] - dataframe["kc_lowerband"]) / # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) # ) # dataframe["kc_width"] = ( # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] # ) # # Ultimate Oscillator # dataframe['uo'] = ta.ULTOSC(dataframe) # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) # RSI dataframe['rsi'] = ta.RSI(dataframe) # # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) # rsi = 0.1 * (dataframe['rsi'] - 50) # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) # # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) # # Stochastic Slow # stoch = ta.STOCH(dataframe) # dataframe['slowd'] = stoch['slowd'] # dataframe['slowk'] = stoch['slowk'] # Stochastic Fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] dataframe['fastk'] = stoch_fast['fastk'] # # Stochastic RSI # Please read https://github.com/freqtrade/freqtrade/issues/2961 before using this. # STOCHRSI is NOT aligned with tradingview, which may result in non-expected results. # stoch_rsi = ta.STOCHRSI(dataframe) # dataframe['fastd_rsi'] = stoch_rsi['fastd'] # dataframe['fastk_rsi'] = stoch_rsi['fastk'] # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] dataframe['macdhist'] = macd['macdhist'] # MFI dataframe['mfi'] = ta.MFI(dataframe) # # ROC # dataframe['roc'] = ta.ROC(dataframe) # Overlap Studies # ------------------------------------ # Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] dataframe["bb_percent"] = ( (dataframe["close"] - dataframe["bb_lowerband"]) / (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) ) dataframe["bb_width"] = ( (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] ) # Bollinger Bands - Weighted (EMA based instead of SMA) # weighted_bollinger = qtpylib.weighted_bollinger_bands( # qtpylib.typical_price(dataframe), window=20, stds=2 # ) # dataframe["wbb_upperband"] = weighted_bollinger["upper"] # dataframe["wbb_lowerband"] = weighted_bollinger["lower"] # dataframe["wbb_middleband"] = weighted_bollinger["mid"] # dataframe["wbb_percent"] = ( # (dataframe["close"] - dataframe["wbb_lowerband"]) / # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) # ) # dataframe["wbb_width"] = ( # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"] # ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) # dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average # dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) # dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) # dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) # dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) # dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) # Parabolic SAR dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) # Cycle Indicator # ------------------------------------ # Hilbert Transform Indicator - SineWave hilbert = ta.HT_SINE(dataframe) dataframe['htsine'] = hilbert['sine'] dataframe['htleadsine'] = hilbert['leadsine'] # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ # # Hammer: values [0, 100] # dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) # # Inverted Hammer: values [0, 100] # dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) # # Dragonfly Doji: values [0, 100] # dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) # # Piercing Line: values [0, 100] # dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] # # Morningstar: values [0, 100] # dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] # # Three White Soldiers: values [0, 100] # dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] # Pattern Recognition - Bearish candlestick patterns # ------------------------------------ # # Hanging Man: values [0, 100] # dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) # # Shooting Star: values [0, 100] # dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) # # Gravestone Doji: values [0, 100] # dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) # # Dark Cloud Cover: values [0, 100] # dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) # # Evening Doji Star: values [0, 100] # dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) # # Evening Star: values [0, 100] # dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) # Pattern Recognition - Bullish/Bearish candlestick patterns # ------------------------------------ # # Three Line Strike: values [0, -100, 100] # dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) # # Spinning Top: values [0, -100, 100] # dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] # # Engulfing: values [0, -100, 100] # dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] # # Harami: values [0, -100, 100] # dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] # # Three Outside Up/Down: values [0, -100, 100] # dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] # # Three Inside Up/Down: values [0, -100, 100] # dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] # # Chart type # # ------------------------------------ # # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] # dataframe['ha_high'] = heikinashi['high'] # dataframe['ha_low'] = heikinashi['low'] # Retrieve best bid and best ask from the orderbook # ------------------------------------ """ # first check if dataprovider is available if self.dp: if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] """ return dataframe def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['enter_short'] = 0 dataframe['enter_long'] = 0 for e in range(1, self.num_episode + 1): logging.info(f'\n------Episode: {e}/{self.num_episode}-------') self.agent.reset() # reset to initial balance and hyperparameters for i in range(0, dataframe.shape[0]): if(i > self.window_size): state = generate_combined_state(i, self.window_size, dataframe.iloc[:, dataframe.columns.get_loc('close')]) action = np.argmax(self.agent.model.predict(state, verbose = 0)[0]) random_action = self.agent.act(state) if action != np.argmax(actions): logging.info(f"\t\t'{self.action_dict[action]}' is an exploration.") if action == 0: # hold execution_result = hold(actions) if action == 1: # buy execution_result = buy(t) if action == 2: # sell execution_result = sell(t) ################ for t in range(1, trading_period + 1): reward = 0 next_state = generate_combined_state(t, window_size, stock_prices, agent.balance, len(agent.inventory)) actions = agent.model.predict(state)[0] action = agent.act(state) if action != np.argmax(actions): logging.info(f"\t\t'{action_dict[action]}' is an exploration.") if action == 0: # hold execution_result = hold(actions) if action == 1: # buy execution_result = buy(t) if action == 2: # sell execution_result = sell(t) # check execution result if execution_result is None: reward -= average_interest_rate() * agent.balance # missing opportunity else: if isinstance(execution_result, tuple): # if execution_result is 'Hold' actions = execution_result[1] execution_result = execution_result[0] logging.info(execution_result) # calculate reward current_portfolio_value = len(agent.inventory) * stock_prices[t] + agent.balance unrealized_profit = current_portfolio_value - agent.initial_portfolio_value reward += unrealized_profit agent.portfolio_values.append(current_portfolio_value) #return daily P/L rates agent.return_rates.append((current_portfolio_value - previous_portfolio_value) / previous_portfolio_value) done = True if t == trading_period else False agent.remember(state, actions, reward, next_state, done) # update state state = next_state # experience replay if len(agent.memory) > agent.buffer_size: num_experience_replay += 1 loss = agent.experience_replay() logging.info('Episode: {}\tLoss: {:.2f}\tAction: {}\tReward: {:.2f}\tBalance: {:.2f}\tNumber of Stocks: {}'.format(e, loss, action_dict[action], reward, agent.balance, len(agent.inventory))) if done: portfolio_return = evaluate_portfolio_performance(agent, logging) returns_across_episodes.append(portfolio_return) # save models every episode agent.model.save('saved_models/DQN_ep' + str(e) + '.h5') logging.info('model saved') plot_portfolio_performance_comparison(stock_name, agent) print("Training Complete") for i in range(0, dataframe.shape[0]): # logger.info(dataframe.iloc[:, dataframe.columns.get_loc('close')]) if(i > self.window_size): state = generate_combined_state(i, self.window_size, dataframe.iloc[:, dataframe.columns.get_loc('close')]) action = np.argmax(self.agent.model.predict(state, verbose = 0)[0]) if action == 1: #Buy print("Buy Taken") dataframe.iloc[i, dataframe.columns.get_loc('enter_long')] = 1 if action == 2: #Sell pass # print("Sell Taken") # dataframe.iloc[i, dataframe.columns.get_loc('enter_short')] = 1 return dataframe def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the exit signal for the given dataframe :param dataframe: DataFrame :param metadata: Additional information, like the currently traded pair :return: DataFrame with exit columns populated """ dataframe.loc[ ( (qtpylib.crossed_above(dataframe['rsi'], self.sell_rsi.value)) & # Signal: RSI crosses above sell_rsi (dataframe['tema'] > dataframe['bb_middleband']) & # Guard: tema above BB middle (dataframe['tema'] < dataframe['tema'].shift(1)) & # Guard: tema is falling (dataframe['volume'] > 0) # Make sure Volume is not 0 ), 'exit_long'] = 1 # Uncomment to use shorts (Only used in futures/margin mode. Check the documentation for more info) """ dataframe.loc[ ( (qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value)) & # Signal: RSI crosses above buy_rsi (dataframe['tema'] <= dataframe['bb_middleband']) & # Guard: tema below BB middle (dataframe['tema'] > dataframe['tema'].shift(1)) & # Guard: tema is raising (dataframe['volume'] > 0) # Make sure Volume is not 0 ), 'exit_short'] = 1 """ return dataframe