{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   
   "source": [
    "[![Script](img/badge-script.svg)](/Teaching//Common.fsx)\u0026emsp;\n",
    "\n"
   ]
  }
,
  {
   "cell_type": "code",
   "metadata": {
    "dotnet_interactive": {
     "language": "fsharp"
    },
    "polyglot_notebook": {
     "kernelName": "fsharp"
    }
   },
   "execution_count": 1, "outputs": [],
   "source": [
    "#r \"nuget: FSharp.Data, 5.0.2\"\n",
    "\n",
    "open System\n",
    "open System.IO\n",
    "open FSharp.Data\n",
    "\n",
    "Environment.CurrentDirectory \u003c- __SOURCE_DIRECTORY__\n",
    "\n",
    "module Secrets =\n",
    "    let envVars = System.Environment.GetEnvironmentVariables() \n",
    "    let localPath = \"../secrets.fsx\"\n",
    "    let localPath2 = \"secrets.fsx\"\n",
    "    let tiingoKey = \n",
    "        let var = \"TIINGO_KEY\"\n",
    "        if envVars.Contains var then \n",
    "            envVars.[var] :?\u003e string\n",
    "        elif File.Exists(localPath) then \n",
    "            File.ReadAllText(localPath)\n",
    "                .Replace(\"let tiingoKey = \",\"\")\n",
    "                .Replace(\"\\\"\",\"\")\n",
    "        elif File.Exists(localPath2) then\n",
    "            File.ReadAllText(localPath2)\n",
    "                .Replace(\"let tiingoKey = \",\"\")\n",
    "                .Replace(\"\\\"\",\"\")\n",
    "        else \"you don\u0027t have a key\"\n",
    "\n",
    "\n",
    "type Frequency = Daily | Monthly\n",
    "type ReturnObs = { Symbol : string; Date : DateTime; Return : float }\n",
    "\n",
    "module Tiingo =\n",
    "\n",
    "    type TiingoCsv = CsvProvider\u003c\"date,close,high,low,open,volume,adjClose,adjHigh,adjLow,adjOpen,adjVolume,divCash,splitFactor\n",
    "2020-10-01,9.77,10.25,9.69,10.09,4554055,9.77,10.25,9.69,10.09,4554055.0,0.0,1.0\"\u003e\n",
    "\n",
    "    type TiingoRequest = { Symbol : string; Start : DateTime; End : DateTime }\n",
    "\n",
    "    type TiingoObs =\n",
    "        {\n",
    "            Date : DateTime\n",
    "            Close : decimal\n",
    "            High : decimal\n",
    "            Low : decimal\n",
    "            Open : decimal \n",
    "            Volume : int\n",
    "            AdjClose : decimal\n",
    "            AdjHigh : decimal\n",
    "            AdjLow : decimal\n",
    "            AdjOpen : decimal\n",
    "            AdjVolume : decimal\n",
    "            DivCash : decimal\n",
    "            SplitFactor : decimal\n",
    "        }\n",
    "\n",
    "    ///\u003csummary\u003eConstructs a Tiingo request. By default is to get the past year of data.\u003c/summary\u003e\n",
    "        /// \u003cparam name=\"symbol\"\u003eThe ticker symbol such as \"AAPL\",\"MSFT\" etc.\u003c/param\u003e\n",
    "    let request symbol = { Symbol = symbol; Start = DateTime.Now.AddYears(-1); End = DateTime.Now}\n",
    "    ///\u003csummary\u003eSets the Tiingo request start date.\u003c/summary\u003e\n",
    "        /// \u003cparam name=\"startOn\"\u003eRequest start date\u003c/param\u003e\n",
    "        /// \u003cparam name=\"request\"\u003eThe Tiingo request to update.\u003c/param\u003e\n",
    "    let startOn startOn request = { request with Start = startOn }\n",
    "    ///\u003csummary\u003eSets the Tiingo request end date.\u003c/summary\u003e\n",
    "        /// \u003cparam name=\"endOn\"\u003eRequest start date\u003c/param\u003e\n",
    "        /// \u003cparam name=\"request\"\u003eThe Tiingo request to update.\u003c/param\u003e\n",
    "    let endOn endOn request = { request with End = endOn }\n",
    "\n",
    "    let private cache = Runtime.Caching.createInMemoryCache (TimeSpan(hours=12,minutes=0,seconds=0))\n",
    "\n",
    "    ///\u003csummary\u003eDownloads Tiingo data.\u003c/summary\u003e\n",
    "        /// \u003cparam name=\"request\"\u003eThe Tiingo request to download.\u003c/param\u003e\n",
    "    let get request =\n",
    "        let dtStr (x : DateTime) = x.Date.ToString(\"yyyy-MM-dd\")\n",
    "        let request = { request with Start = request.Start.Date; End = request.End.Date }\n",
    "        let key = $\"{request.Symbol}-{dtStr request.Start}-{dtStr request.End}.csv\"\n",
    "        match cache.TryRetrieve(key) with\n",
    "        | Some res -\u003e res\n",
    "        | None -\u003e\n",
    "            let result = \n",
    "                Http.RequestString\n",
    "                            ( $\"https://api.tiingo.com/tiingo/daily/{request.Symbol}/prices\", \n",
    "                                httpMethod = \"GET\",\n",
    "                                query   = [ \"token\", Secrets.tiingoKey; \n",
    "                                            \"startDate\", request.Start.ToString(\"yyyy-MM-dd\");\n",
    "                                            \"endDate\", request.End.ToString(\"yyyy-MM-dd\");\n",
    "                                            \"format\",\"csv\"],\n",
    "                                headers = [HttpRequestHeaders.Accept HttpContentTypes.Csv])\n",
    "            cache.Set(key,result)\n",
    "            result\n",
    "        |\u003e TiingoCsv.Parse\n",
    "        |\u003e fun parsed -\u003e\n",
    "            parsed.Rows\n",
    "            |\u003e Seq.map(fun row -\u003e\n",
    "                { Date = row.Date\n",
    "                  Close = row.Close\n",
    "                  High = row.High\n",
    "                  Low = row.Low\n",
    "                  Open = row.Open\n",
    "                  Volume = row.Volume\n",
    "                  AdjClose = row.AdjClose\n",
    "                  AdjHigh = row.AdjHigh\n",
    "                  AdjLow = row.AdjLow\n",
    "                  AdjOpen = row.AdjOpen\n",
    "                  AdjVolume = row.AdjVolume\n",
    "                  DivCash = row.DivCash\n",
    "                  SplitFactor = row.SplitFactor \n",
    "                  })\n",
    "            |\u003e Seq.toArray      \n",
    "    \n",
    "    // using a class, keeping private for now.\n",
    "    type private Download(symbol:string,?startOn:DateTime,?endOn:DateTime) =\n",
    "        let startOn = defaultArg startOn (DateTime.Now.AddYears(-1))\n",
    "        let endOn = defaultArg endOn (DateTime.Now)\n",
    "        let data = get { Symbol = symbol; Start = startOn; End = endOn }\n",
    "        member this.Rows = data\n",
    " \n",
    "    // Probably deprecated\n",
    "    let private getFromCacheDirectory cacheDirectory request =\n",
    "        let dtStr (x : DateTime) = x.Date.ToString(\"yyyy-MM-dd\")\n",
    "        let request = { request with Start = request.Start.Date; End = request.End.Date }\n",
    "        let key = $\"{request.Symbol}-{dtStr request.Start}-{dtStr request.End}.csv\"\n",
    "        let cacheFile = cacheDirectory + key\n",
    "        if File.Exists(cacheFile) then\n",
    "            File.ReadAllText(cacheFile)\n",
    "        else    \n",
    "            let result = \n",
    "                Http.RequestString\n",
    "                            ( $\"https://api.tiingo.com/tiingo/daily/{request.Symbol}/prices\", \n",
    "                                httpMethod = \"GET\",\n",
    "                                query   = [ \"token\", Secrets.tiingoKey; \n",
    "                                            \"startDate\", request.Start.ToString(\"yyyy-MM-dd\");\n",
    "                                            \"endDate\", request.End.ToString(\"yyyy-MM-dd\");\n",
    "                                            \"format\",\"csv\"],\n",
    "                                headers = [HttpRequestHeaders.Accept HttpContentTypes.Csv])\n",
    "            File.WriteAllText(cacheFile,result)\n",
    "            result\n",
    "        |\u003e TiingoCsv.Parse\n",
    "    \n",
    "    let private returnHelper symbol (xs:TiingoObs seq) =\n",
    "        xs\n",
    "        |\u003e Seq.sortBy(fun x -\u003e x.Date)\n",
    "        |\u003e Seq.pairwise\n",
    "        |\u003e Seq.map(fun (yesterday, today) -\u003e\n",
    "            { Symbol = symbol \n",
    "              Date = today.Date\n",
    "              Return =  float (today.AdjClose / yesterday.AdjClose) - 1.0})\n",
    "        |\u003e Seq.toArray      \n",
    "\n",
    "    let getReturns request =\n",
    "        get request\n",
    "        |\u003e (returnHelper request.Symbol)\n",
    "\n",
    "    // Marking as private so people don\u0027t use it by accident\n",
    "    let private getInternetFileCache request =\n",
    "        let cache = Runtime.Caching.createInternetFileCache \"tiingo\" (TimeSpan.FromDays 30.0)\n",
    "        let request = { request with Start = request.Start.Date; End = request.End.Date }\n",
    "        let key = request.ToString()\n",
    "        match cache.TryRetrieve(key) with\n",
    "        | Some res -\u003e res\n",
    "        | None -\u003e\n",
    "            let res =\n",
    "                Http.RequestString\n",
    "                        ( $\"https://api.tiingo.com/tiingo/daily/{request.Symbol}/prices\", \n",
    "                            httpMethod = \"GET\",\n",
    "                            query   = [ \"token\", Secrets.tiingoKey; \n",
    "                                        \"startDate\", request.Start.ToString(\"yyyy-MM-dd\");\n",
    "                                        \"endDate\", request.End.ToString(\"yyyy-MM-dd\");\n",
    "                                        \"format\",\"csv\"],\n",
    "                            headers = [HttpRequestHeaders.Accept HttpContentTypes.Csv ])\n",
    "            cache.Set(key, res)\n",
    "            res\n",
    "        |\u003e TiingoCsv.Parse\n",
    "\n",
    "module French =\n",
    "    //open System.Net\n",
    "    open System.IO.Compression\n",
    "\n",
    "    type private FF3Csv = CsvProvider\u003c\"Date (string),Mkt-RF,SMB,HML,RF\n",
    "        19260701,    0.10,   -0.24,   -0.28,   0.009\"\u003e\n",
    "    type FF3Obs = \n",
    "        { Date : DateTime \n",
    "          MktRf : float\n",
    "          Smb : float \n",
    "          Hml : float\n",
    "          Rf : float \n",
    "          Frequency : Frequency } \n",
    "\n",
    "    type private FF5Csv = CsvProvider\u003c\"Date (string),Mkt-RF,SMB,HML,RMW,CMA,RF\n",
    "        19260701,    0.10,   -0.24,   -0.28,0.0,1.2,  0.009\"\u003e\n",
    "\n",
    "    type FF5Obs = \n",
    "        { Date : DateTime \n",
    "          MktRf : float\n",
    "          Smb : float \n",
    "          Hml : float\n",
    "          Rmw : float\n",
    "          Cma : float\n",
    "          Rf : float \n",
    "          Frequency : Frequency } \n",
    "\n",
    "    let private frenchDay x = \n",
    "        DateTime.ParseExact(x,\n",
    "            \"yyyyMMdd\",\n",
    "            Globalization.CultureInfo.InvariantCulture)\n",
    "    let private frenchMonth x = \n",
    "        DateTime.ParseExact(x,\n",
    "            \"yyyyMM\",\n",
    "            Globalization.CultureInfo.InvariantCulture)\n",
    "\n",
    "    let private cache = \n",
    "        let today = DateTime.Now\n",
    "        let nextMonth = today.AddMonths(1)\n",
    "        let eom = DateTime(nextMonth.Year, nextMonth.Month, 1).AddSeconds(-1.0) \n",
    "        Runtime.Caching.createInternetFileCache \"French\" (eom - today)\n",
    "\n",
    "    let private getData (dataset:string) =\n",
    "        match cache.TryRetrieve(dataset) with\n",
    "        | Some data -\u003e data\n",
    "        | None -\u003e\n",
    "            //let dataset = \"F-F_Research_Data_Factors_CSV\"\n",
    "            let urlString = $\"http://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/{dataset}.zip\"\n",
    "            let request = Http.RequestStream(urlString, httpMethod = \"GET\",headers = [HttpRequestHeaders.Accept HttpContentTypes.Any])\n",
    "            use archive = new ZipArchive(request.ResponseStream,ZipArchiveMode.Read)\n",
    "            let file = archive.GetEntry($\"{dataset}\".Replace(\"_CSV\",\".CSV\"))\n",
    "            use reader = new StreamReader(file.Open())\n",
    "            let data  = reader.ReadToEnd()\n",
    "            cache.Set(dataset,data)\n",
    "            data\n",
    "    let getFF3 frequency =\n",
    "            let (dataset, dateParser) =\n",
    "                match frequency with\n",
    "                | Monthly -\u003e \"F-F_Research_Data_Factors_CSV\", frenchMonth\n",
    "                | Daily -\u003e \"F-F_Research_Data_Factors_daily_CSV\", frenchDay\n",
    "            let data = new StringReader(getData dataset)\n",
    "            [| while data.Peek() \u003c\u003e -1 do\n",
    "                    data.ReadLine() |]\n",
    "            |\u003e Array.skipWhile(fun line -\u003e not (line.Contains(\"Mkt-RF\")))\n",
    "            |\u003e Array.skip 1\n",
    "            |\u003e Array.takeWhile(fun line -\u003e line \u003c\u003e \"\")\n",
    "            |\u003e Array.map(fun line -\u003e \n",
    "                let parsedLine = FF3Csv.ParseRows(line).[0] \n",
    "                { Date = dateParser parsedLine.Date\n",
    "                  MktRf = float parsedLine.``Mkt-RF`` / 100.0\n",
    "                  Smb = float parsedLine.SMB / 100.0\n",
    "                  Hml = float parsedLine.HML / 100.0\n",
    "                  Rf = float parsedLine.RF / 100.0 \n",
    "                  Frequency = frequency })\n",
    "\n",
    "    let getFF5 frequency =\n",
    "        let (dataset, dateParser) =\n",
    "            match frequency with\n",
    "            | Monthly -\u003e \"F-F_Research_Data_5_Factors_2x3_CSV\", frenchMonth\n",
    "            | Daily -\u003e \"F-F_Research_Data_5_Factors_2x3_daily_CSV\", frenchDay\n",
    "        let data = new StringReader(getData dataset)\n",
    "        [| while data.Peek() \u003c\u003e -1 do\n",
    "                data.ReadLine() |]\n",
    "        |\u003e Array.skipWhile(fun line -\u003e not (line.Contains(\"Mkt-RF\")))\n",
    "        |\u003e Array.skip 1\n",
    "        |\u003e Array.takeWhile(fun line -\u003e line \u003c\u003e \"\")\n",
    "        |\u003e Array.map(fun line -\u003e \n",
    "            let parsedLine = FF5Csv.ParseRows(line).[0] \n",
    "            { Date = dateParser parsedLine.Date\n",
    "              MktRf = float parsedLine.``Mkt-RF`` / 100.0\n",
    "              Smb = float parsedLine.SMB / 100.0\n",
    "              Hml = float parsedLine.HML / 100.0\n",
    "              Rmw = float parsedLine.RMW / 100.0\n",
    "              Cma = float parsedLine.CMA / 100.0\n",
    "              Rf = float parsedLine.RF / 100.0 \n",
    "              Frequency = frequency })\n",
    "\n",
    "module Fred =\n",
    "    type Series = CsvProvider\u003c\"https://fred.stlouisfed.org/graph/fredgraph.csv?id=GS10\",\n",
    "                              Schema=\"Date,Value (float)\",\n",
    "                              MissingValues=\".\"\u003e\n",
    "    let private fredUrl series = $\"https://fred.stlouisfed.org/graph/fredgraph.csv?id={series}\"\n",
    "    \n",
    "    ///\u003csummary\u003eGets a FRED data series as a CsvProvider\u003c/summary\u003e\n",
    "        /// \u003cparam name=\"series\"\u003eThe series name such as GS10, EXUSEU, etc.\u003c/param\u003e\n",
    "    let get (series:string) =  Series.Load(fredUrl series)\n",
    "    \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".NET (F#)",
   "language": "F#",
   "name": ".net-fsharp"
  },
  "language_info": {
   "file_extension": ".fs",
   "mimetype": "text/x-fsharp",
   "name": "polyglot-notebook",
   "pygments_lexer": "fsharp"
  },
  "polyglot_notebook": {
   "kernelInfo": {
    "defaultKernelName": "fsharp",
    "items": [
     {
      "aliases": [],
      "languageName": "fsharp",
      "name": "fsharp"
     }
    ]
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}