{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%pip install pandas==0.24.1 --user\n", "%pip install tensorboardX --user\n", "%pip install bs4 --user\n", "%pip install -U auquan_toolbox --user" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "WsNIiolz3r9C" }, "source": [ "### This notebook shows how Auquan Toolbox can be used to trade on momentum and mean reversion\n", "\n", "Documentation on how to use the toolbox can be found [here](https://github.com/Auquan/auquan-toolbox-python#3-backtesting). " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 381 }, "colab_type": "code", "id": "IjSoh14b3r9G", "outputId": "e8afb1bc-8564-47ff-a067-f5970d7fa3d3" }, "outputs": [], "source": [ "from backtester.trading_system_parameters import TradingSystemParameters\n", "from backtester.features.feature import Feature\n", "from backtester.dataSource.yahoo_data_source import YahooStockDataSource\n", "from backtester.timeRule.custom_time_rule import CustomTimeRule\n", "from backtester.executionSystem.simple_execution_system import SimpleExecutionSystem\n", "from backtester.orderPlacer.backtesting_order_placer import BacktestingOrderPlacer\n", "from backtester.trading_system import TradingSystem\n", "from backtester.constants import *\n", "\n", "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import datetime\n", "from datetime import timedelta" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "yaqazil43r9P" }, "source": [ "### This is the static part of an experiment\n", "\n", "It is similar to writing a data loader and trainer for a deep learning project. Once written for a particular experiment we hardly need to change it." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", "id": "YBS5-alV3r9S" }, "outputs": [], "source": [ "class MyTradingParams(TradingSystemParameters):\n", "\n", " def __init__(self, tradingFunctions):\n", " self.__tradingFunctions = tradingFunctions\n", " super(MyTradingParams, self).__init__()\n", " self.__dataSetId = 'equity_data'\n", " self.__instrumentIds = self.__tradingFunctions.getSymbolsToTrade()\n", " self.__startDate = '2015/01/02'\n", " self.__endDate = '2017/08/31'\n", "\n", " def getDataParser(self):\n", " '''\n", " Returns an instance of class DataParser. Source of data for instruments\n", " '''\n", " instrumentIds = self.__tradingFunctions.getSymbolsToTrade()\n", " return YahooStockDataSource(\n", " cachedFolderName = 'historicalData/',\n", " dataSetId = self.__dataSetId,\n", " instrumentIds = instrumentIds,\n", " startDateStr = self.__startDate,\n", " endDateStr = self.__endDate,\n", " )\n", " \n", " def getTimeRuleForUpdates(self):\n", " return CustomTimeRule(\n", " startDate = self.__startDate, \n", " endDate = self.__endDate, \n", " frequency = 'D', \n", " sample = '30'\n", " )\n", "\n", " def getFrequencyOfFeatureUpdates(self):\n", " return timedelta(days = 1)\n", "\n", " def getStartingCapital(self):\n", " if len(self.__tradingFunctions.getSymbolsToTrade()) > 0:\n", " return 1000*len(self.__tradingFunctions.getSymbolsToTrade())\n", " else:\n", " return 30000\n", "\n", " def getCustomFeatures(self):\n", " '''\n", " This is a way to use any custom features you might have made.\n", " Returns a dictionary where:\n", " \n", " key: featureId to access this feature (Make sure this doesnt conflict with any of the pre defined feature Ids)\n", " value: Your custom Class which computes this feature. The class should be an instance of Feature\n", " \n", " Eg. if your custom class is MyCustomFeature, and you want to access this via featureId='my_custom_feature',\n", " you will import that class, and return this function as {'my_custom_feature': MyCustomFeature}\n", " '''\n", " return {\n", " 'my_custom_feature': MyCustomFeature,\n", " 'prediction': TrainingPredictionFeature,\n", " 'zero_fees': FeesCalculator,\n", " 'benchmark_PnL': BuyHoldPnL,\n", " 'score': ScoreFeature\n", " }\n", "\n", " def getInstrumentFeatureConfigDicts(self):\n", " '''\n", " Returns an array of instrument feature config dictionaries instrument feature config Dictionary has the \n", " following keys:\n", " \n", " featureId: a string representing the type of feature you want to use\n", " featureKey: a string representing the key you will use to access the value of this feature\n", " params: A dictionary with which contains other optional params if needed by the feature\n", " '''\n", "\n", " predictionDict = {\n", " 'featureKey': 'prediction',\n", " 'featureId': 'prediction',\n", " 'params': {}\n", " }\n", " feesConfigDict = {\n", " 'featureKey': 'fees',\n", " 'featureId': 'zero_fees',\n", " 'params': {}\n", " }\n", " profitlossConfigDict = {\n", " 'featureKey': 'pnl',\n", " 'featureId': 'pnl',\n", " 'params': {\n", " 'price': self.getPriceFeatureKey(),\n", " 'fees': 'fees'\n", " }\n", " }\n", " capitalConfigDict = {\n", " 'featureKey': 'capital',\n", " 'featureId': 'capital',\n", " 'params': {\n", " 'price': 'adjClose',\n", " 'fees': 'fees',\n", " 'capitalReqPercent': 0.95\n", " }\n", " }\n", " benchmarkDict = {\n", " 'featureKey': 'benchmark',\n", " 'featureId': 'benchmark_PnL',\n", " 'params': {'pnlKey': 'pnl'}\n", " }\n", " scoreDict = {\n", " 'featureKey': 'score',\n", " 'featureId': 'score',\n", " 'params': {\n", " 'featureName1': 'pnl',\n", " 'featureName2':'benchmark'\n", " }\n", " }\n", "\n", " stockFeatureConfigs = self.__tradingFunctions.getInstrumentFeatureConfigDicts()\n", "\n", " return {\n", " INSTRUMENT_TYPE_STOCK: stockFeatureConfigs + [\n", " predictionDict,\n", " feesConfigDict,\n", " profitlossConfigDict,\n", " capitalConfigDict,\n", " benchmarkDict, \n", " scoreDict\n", " ]\n", " }\n", "\n", " def getMarketFeatureConfigDicts(self):\n", " '''\n", " Returns an array of market feature config dictionaries having the following keys:\n", " \n", " featureId: a string representing the type of feature you want to use\n", " featureKey: a string representing the key you will use to access the value of this feature\n", " params: A dictionary with which contains other optional params if needed by the feature\n", " '''\n", " scoreDict = {\n", " 'featureKey': 'score',\n", " 'featureId': 'score_ll',\n", " 'params': {\n", " 'featureName': self.getPriceFeatureKey(),\n", " 'instrument_score_feature': 'score'\n", " }\n", " }\n", " \n", " return [scoreDict]\n", "\n", " def getPrediction(self, time, updateNum, instrumentManager):\n", " predictions = pd.Series(index = self.__instrumentIds)\n", " predictions = self.__tradingFunctions.getPrediction(time, updateNum, instrumentManager, predictions)\n", " \n", " return predictions\n", "\n", " def getExecutionSystem(self):\n", " '''\n", " Returns the type of execution system we want to use. Its an implementation of the class ExecutionSystem\n", " It converts prediction to intended positions for different instruments.\n", " '''\n", " \n", " return SimpleExecutionSystem(\n", " enter_threshold = 0.7,\n", " exit_threshold = 0.55,\n", " longLimit = 1,\n", " shortLimit = 1,\n", " capitalUsageLimit = 0.10*self.getStartingCapital(),\n", " enterlotSize = 1, \n", " exitlotSize = 1,\n", " limitType = 'L', \n", " price = 'adjClose'\n", " )\n", "\n", " def getOrderPlacer(self):\n", " '''\n", " Returns the type of order placer we want to use. It's an implementation of the class OrderPlacer.\n", " It helps place an order, and also read confirmations of orders being placed.\n", " For Backtesting, you can just use the BacktestingOrderPlacer, which places the order which you want, and \n", " automatically confirms it too.\n", " '''\n", " \n", " return BacktestingOrderPlacer()\n", "\n", " def getLookbackSize(self):\n", " '''\n", " Returns the amount of lookback data you want for your calculations. The historical market features and instrument features are only\n", " stored upto this amount.\n", " This number is the number of times we have updated our features.\n", " '''\n", " \n", " return 120\n", "\n", " def getPriceFeatureKey(self):\n", " '''\n", " The name of column containing the instrument price\n", " '''\n", " \n", " return 'adjClose'\n", "\n", " def getInstrumentsIds(self):\n", " '''\n", " Get all instrument ids\n", " '''\n", " \n", " return self.__instrumentIds" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Hkb02Y1i3r9b" }, "source": [ "### Let's define some of our own features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", "id": "8EmCAvit3r9e" }, "outputs": [], "source": [ "class TrainingPredictionFeature(Feature):\n", " \n", " @classmethod\n", " def computeForInstrument(cls, updateNum, time, featureParams, featureKey, instrumentManager):\n", " tf = MyTradingFunctions()\n", " t = MyTradingParams(tf)\n", " \n", " return t.getPrediction(time, updateNum, instrumentManager)\n", "\n", "class FeesCalculator(Feature):\n", " @classmethod\n", " def computeForInstrument(cls, updateNum, time, featureParams, featureKey, instrumentManager):\n", " \n", " return pd.Series(0, index = instrumentManager.getAllInstrumentsByInstrumentId())\n", "\n", "class BuyHoldPnL(Feature):\n", " \n", " @classmethod\n", " def computeForInstrument(cls, updateNum, time, featureParams, featureKey, instrumentManager):\n", " instrumentLookbackData = instrumentManager.getLookbackInstrumentFeatures()\n", "\n", " priceData = instrumentLookbackData.getFeatureDf('adjClose')\n", " \n", " if len(priceData) < 2:\n", " return pd.Series(0, index = instrumentManager.getAllInstrumentsByInstrumentId())\n", " else:\n", " bhpnl = instrumentLookbackData.getFeatureDf(featureKey).iloc[-1]\n", " bhpnl += priceData.iloc[-1] - priceData.iloc[-2]\n", "\n", " return bhpnl\n", "\n", "class ScoreFeature(Feature):\n", " \n", " @classmethod\n", " def computeForInstrument(cls, updateNum, time, featureParams, featureKey, instrumentManager):\n", " instrumentLookbackData = instrumentManager.getLookbackInstrumentFeatures()\n", " if len(instrumentLookbackData.getFeatureDf(featureParams['featureName1'])) > 0:\n", " feature1 = instrumentLookbackData.getFeatureDf(featureParams['featureName1']).iloc[-1]\n", " feature2 = instrumentLookbackData.getFeatureDf(featureParams['featureName2']).iloc[-1]\n", " \n", " for instrumentId in feature1.index:\n", " pnls = instrumentLookbackData.getFeatureDf('pnl')[instrumentId]\n", " positions = instrumentLookbackData.getFeatureDf('position')[instrumentId]\n", " \n", " print(instrumentId)\n", " print('pnl: %.2f'%pnls[-1])\n", " if len(positions) > 2 and np.abs(positions[-1] - positions[-2]) > 0:\n", " print('Position changed to: %.2f'%positions[-1])\n", " \n", " toRtn = (feature1 - feature2) / feature2.abs()\n", " toRtn[toRtn.isnull()] = 0\n", " toRtn[toRtn == np.Inf] = 0\n", " else:\n", " toRtn=0\n", "\n", " return toRtn" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "_YeUN2jV3r9m" }, "source": [ "### This is the part where the magic takes place, all the logic for prediction and carrying out trades goes here" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", "id": "TmmI1xdA3r9o" }, "outputs": [], "source": [ "class MyTradingFunctions():\n", "\n", " def __init__(self):\n", " self.count = 0\n", " self.params = {}\n", "\n", " def getSymbolsToTrade(self):\n", " '''\n", " Specify the stock names that you want to trade.\n", " '''\n", " \n", " return ['AAPL']\n", "\n", " def getInstrumentFeatureConfigDicts(self):\n", " '''\n", " Specify all Features you want to use by creating config dictionaries.\n", " Create one dictionary per feature and return them in an array.\n", "\n", " Feature config Dictionary have the following keys:\n", "\n", " featureId: a str for the type of feature you want to use\n", " featureKey: {optional} a str for the key you will use to call this feature\n", " If not present, will just use featureId\n", " params: {optional} A dictionary with which contains other optional params if needed by the feature\n", "\n", " msDict = {\n", " 'featureKey': 'ms_5',\n", " 'featureId': 'moving_sum',\n", " 'params': {\n", " 'period': 5,\n", " 'featureName': 'basis'\n", " }\n", " }\n", "\n", " return [msDict]\n", "\n", " You can now use this feature by in getPRediction() calling it's featureKey, 'ms_5'\n", " '''\n", "\n", " ma1Dict = {\n", " 'featureKey': 'ma_90',\n", " 'featureId': 'moving_average',\n", " 'params': {\n", " 'period': 90,\n", " 'featureName': 'adjClose'\n", " }\n", " }\n", " mom30Dict = {\n", " 'featureKey': 'mom_30',\n", " 'featureId': 'momentum',\n", " 'params': {\n", " 'period': 30,\n", " 'featureName': 'adjClose'\n", " }\n", " }\n", " mom10Dict = {\n", " 'featureKey': 'mom_10',\n", " 'featureId': 'momentum',\n", " 'params': {\n", " 'period': 10,\n", " 'featureName': 'adjClose'\n", " }\n", " }\n", " \n", " return [ma1Dict, mom10Dict, mom30Dict]\n", "\n", " def getPrediction(self, time, updateNum, instrumentManager, predictions):\n", " '''\n", " Combine all the features to create the desired predictions for each stock.\n", " 'predictions' is Pandas Series with stock as index and predictions as values\n", " We first call the holder for all the instrument features for all stocks as\n", " lookbackInstrumentFeatures = instrumentManager.getLookbackInstrumentFeatures()\n", " Then call the dataframe for a feature using its feature_key as\n", " ms5Data = lookbackInstrumentFeatures.getFeatureDf('ms_5')\n", " This returns a dataFrame for that feature for ALL stocks for all times upto lookback time\n", " Now you can call just the last data point for ALL stocks as\n", " ms5 = ms5Data.iloc[-1]\n", " You can call last datapoint for one stock 'ABC' as\n", " value_for_abs = ms5['ABC']\n", "\n", " Output of the prediction function is used by the toolbox to make further trading decisions and evaluate your score.\n", " '''\n", "\n", " # self.updateCount() - uncomment if you want a counter\n", "\n", " # holder for all the instrument features for all instruments\n", " lookbackInstrumentFeatures = instrumentManager.getLookbackInstrumentFeatures()\n", "\n", " #############################################################################################\n", " ### TODO : FILL THIS FUNCTION TO RETURN A BUY (1) or SELL (0) prediction for each stock ###\n", " ### USE TEMPLATE BELOW AS EXAMPLE\n", " ###\n", " ### HINT: Use the Hurst Exponent \n", " ### http://analytics-magazine.org/the-hurst-exponent-predictability-of-time-series/\n", " #############################################################################################\n", " \n", " # Here's an example implementation of the hurst exponent\n", " def hurst_f(input_ts, lags_to_test=20): \n", " # interpretation of return value\n", " # hurst < 0.5 - input_ts is mean reverting\n", " # hurst = 0.5 - input_ts is effectively random/geometric brownian motion\n", " # hurst > 0.5 - input_ts is trending\n", " tau = []\n", " lagvec = [] \n", " # Step through the different lags \n", " for lag in range(2, lags_to_test): \n", " # produce price difference with lag \n", " pp = np.subtract(input_ts[lag:], input_ts[:-lag]) \n", " # Write the different lags into a vector \n", " lagvec.append(lag) \n", " # Calculate the variance of the differnce vector \n", " tau.append(np.sqrt(np.std(pp))) \n", " # linear fit to double-log graph (gives power) \n", " m = np.polyfit(np.log10(lagvec), np.log10(tau), 1) \n", " # calculate hurst \n", " hurst = m[0]*2\n", "\n", " return hurst \n", "\n", " # dataframe for a historical instrument feature (ma_90 in this case). The index is the timestamps\n", " # of upto lookback data points. The columns of this dataframe are the stock symbols/instrumentIds.\n", " mom10Data = lookbackInstrumentFeatures.getFeatureDf('mom_10')\n", " mom30Data = lookbackInstrumentFeatures.getFeatureDf('mom_30')\n", " ma90Data = lookbackInstrumentFeatures.getFeatureDf('ma_90')\n", " \n", " # Here we are making predictions on the basis of Hurst exponent if enough data is available, otherwise\n", " # we simply get out of our position\n", " if len(ma90Data.index)>20:\n", " mom30 = mom30Data.iloc[-1]\n", " mom10 = mom10Data.iloc[-1]\n", " ma90 = ma90Data.iloc[-1]\n", " \n", " # Calculate Hurst Exponent\n", " hurst = ma90Data.apply(hurst_f, axis=0)\n", " # Go long if Hurst > 0.5 and both long term and short term momentum are positive\n", " predictions[(hurst > 0.5) & (mom30 > 0) & (mom10 > 0)] = 1 \n", " # Go short if Hurst > 0.5 and both long term and short term momentum are negative\n", " predictions[(hurst > 0.5) & (mom30 <= 0) & (mom10 <= 0)] = 0 \n", " \n", " # Get out of position if Hurst > 0.5 and long term momentum is positive while short term is negative\n", " predictions[(hurst > 0.5) & (mom30 > 0) & (mom10 <= 0)] = 0.5\n", " # Get out of position if Hurst > 0.5 and long term momentum is negative while short term is positive\n", " predictions[(hurst > 0.5) & (mom30 <= 0) & (mom10 > 0)] = 0.5\n", " \n", " # Get out of position if Hurst < 0.5\n", " predictions[hurst <= 0.5] = 0.5 \n", " else:\n", " # If no sufficient data then don't take any positions\n", " predictions.values[:] = 0.5\n", " return predictions\n", "\n", " def updateCount(self):\n", " self.count = self.count + 1" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "Gp_ajtSC3r9t" }, "source": [ "#### Here's another example of a custom feature" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", "id": "weJEUniK3r9v" }, "outputs": [], "source": [ "class MyCustomFeature(Feature):\n", " ''''\n", " Custom Feature to implement for instrument. This function would return the value of the feature you want to implement.\n", " 1. create a new class MyCustomFeatureClassName for the feature and implement your logic in the function computeForInstrument() -\n", "\n", " 2. modify function getCustomFeatures() to return a dictionary with Id for this class\n", " (follow formats like {'my_custom_feature_identifier': MyCustomFeatureClassName}.\n", " Make sure 'my_custom_feature_identifier' doesnt conflict with any of the pre defined feature Ids\n", "\n", " def getCustomFeatures(self):\n", " return {'my_custom_feature_identifier': MyCustomFeatureClassName}\n", "\n", " 3. create a dict for this feature in getInstrumentFeatureConfigDicts() above. Dict format is:\n", " customFeatureDict = {'featureKey': 'my_custom_feature_key',\n", " 'featureId': 'my_custom_feature_identifier',\n", " 'params': {'param1': 'value1'}}\n", " You can now use this feature by calling it's featureKey, 'my_custom_feature_key' in getPrediction()\n", " '''\n", " \n", " @classmethod\n", " def computeForInstrument(cls, updateNum, time, featureParams, featureKey, instrumentManager):\n", " # Custom parameter which can be used as input to computation of this feature\n", " param1Value = featureParams['param1']\n", "\n", " # A holder for the all the instrument features\n", " lookbackInstrumentFeatures = instrumentManager.getLookbackInstrumentFeatures()\n", "\n", " # dataframe for a historical instrument feature (basis in this case). The index is the timestamps\n", " # atmost upto lookback data points. The columns of this dataframe are the stocks/instrumentIds.\n", " lookbackInstrumentValue = lookbackInstrumentFeatures.getFeatureDf('adjClose')\n", "\n", " # The last row of the previous dataframe gives the last calculated value for that feature (basis in this case)\n", " # This returns a series with stocks/instrumentIds as the index.\n", " currentValue = lookbackInstrumentValue.iloc[-1]\n", "\n", " if param1Value == 'value1':\n", " return currentValue * 0.1\n", " else:\n", " return currentValue * 0.5" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "FAoigNUv3r93" }, "source": [ "### Time to run the backtester!" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", "id": "LMW8EmrC3r95", "scrolled": true }, "outputs": [], "source": [ "tf = MyTradingFunctions()\n", "tsParams = MyTradingParams(tf)\n", "tradingSystem = TradingSystem(tsParams)\n", "results = tradingSystem.startTrading()" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "yCjIlVz-3r99" }, "source": [ "Results for each timestamp are stored as csv file inside the folder `./runLogs`, we also create logs using tensorboardX inside `./tb_logs` so have a look at that as well using `tensorboard --logdir=tb_logs`" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", "id": "l9gAWcy13r9-" }, "outputs": [], "source": [ "results" ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "momentum_backtest_losing_money.ipynb", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.3" } }, "nbformat": 4, "nbformat_minor": 4 }