{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installing packages:\n",
      "\t.package(url: \"https://github.com/mxcl/Path.swift\", from: \"0.16.1\")\n",
      "\t\tPath\n",
      "With SwiftPM flags: []\n",
      "Working in: /tmp/tmpag006_ed/swift-install\n",
      "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "warning: /home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n",
      "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[1/10] Compiling Path Path+Codable.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[2/10] Compiling Path Path+Attributes.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[3/10] Compiling Path Path+CommonDirectories.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[4/10] Compiling Path Extensions.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[5/10] Compiling Path Path->Bool.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[6/10] Compiling Path Path+StringConvertibles.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[7/10] Compiling Path Path.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[8/10] Compiling Path Path+ls.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[9/10] Compiling Path Path+FileManager.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[10/11] Merging module Path\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)[11/12] Compiling jupyterInstalledPackages jupyterInstalledPackages.swift\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "[12/13] Merging module jupyterInstalledPackages\n",
      "/home/sgugger/swift/usr/bin/swift: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift)\n",
      "/home/sgugger/swift/usr/bin/swiftc: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swiftc)\n",
      "/home/sgugger/swift/usr/bin/swift-autolink-extract: /home/sgugger/anaconda3/lib/libuuid.so.1: no version information available (required by /home/sgugger/swift/usr/bin/swift-autolink-extract)\n",
      "[13/13] Linking libjupyterInstalledPackages.so\n",
      "Initializing Swift...\n",
      "Installation complete!\n"
     ]
    }
   ],
   "source": [
    "%install-location $cwd/swift-install\n",
    "%install '.package(url: \"https://github.com/mxcl/Path.swift\", from: \"0.16.1\")' Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import Foundation\n",
    "import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/sgugger/git/swiftai\r\n"
     ]
    }
   ],
   "source": [
    "let path = Path.cwd.parent\n",
    "print(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "public extension String {\n",
    "    func findFirst(pat: String) -> Range<String.Index>? {\n",
    "        return range(of: pat, options: .regularExpression)\n",
    "    }\n",
    "    func hasMatch(pat: String) -> Bool {\n",
    "        return findFirst(pat:pat) != nil\n",
    "    }\n",
    "    func withMaj() -> String {\n",
    "        return prefix(1).capitalized + dropFirst()\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook to script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func nbNameToScriptName(_ name: String) -> String {\n",
    "    var splits = name.components(separatedBy: \"_\")\n",
    "    splits = splits[1...].map { $0.withMaj() }\n",
    "    return splits.joined(separator: \"\")\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func readNb(_ fname: Path) -> [String:Any] {\n",
    "    let data = try! Data(contentsOf: fname.url)\n",
    "    return try! JSONSerialization.jsonObject(with: data, options: .allowFragments) as! [String: Any]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func notebookToScript(_ fname: Path, dest: Path?=nil) {\n",
    "    let newName = nbNameToScriptName(fname.basename(dropExtension: true))+\".swift\"\n",
    "    let destFname = (dest ?? fname.parent) / newName\n",
    "    let cells = readNb(fname)[\"cells\"] as! [[String:Any]]\n",
    "        var module = \"\"\"\n",
    "/*\n",
    "This file was autogenerated from \\(fname.basename())\n",
    "        \n",
    "If you edit it, be sure that:\n",
    "  1. there is no diff between this file and the corresponding notebook prior to editing\n",
    "  2. you don't touch the comments looking like // cell ## as it would break the way back to the notebook\n",
    "        \n",
    "Run *** when you are done to update the notebooks with your change.\n",
    "*/\n",
    "        \n",
    "\"\"\"\n",
    "    for (i,cell) in cells.enumerated() {\n",
    "        if let source = cell[\"source\"] as? [String], !source.isEmpty, source[0].hasMatch(pat: #\"^\\s*//\\s*export\\s*$\"#) {\n",
    "            module.append(\"\\n//cell\\(i)\\n\\(source[1...].joined())\\n\")\n",
    "        }\n",
    "    }\n",
    "    try! module.write(to: destFname, encoding: .utf8)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "let fname = path/\"/nbs/stable/00_load_data.ipynb\"\n",
    "let dest = path/\"Sources/SwiftAI\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "notebookToScript(fname, dest: dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func makeLibrary(_ nbFolder: Path, dest: Path?=nil){\n",
    "    for entry in try! nbFolder.ls()  where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #\"^\\d+[a-z]*_.*ipynb$\"#) {\n",
    "        print(\"Converting \\(entry.path.basename())\")\n",
    "        notebookToScript(entry.path, dest: dest)\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "let nbFolder = path/\"/nbs/stable\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting 02a_why_sqrt5.ipynb\n",
      "Converting 10_mixup_ls.ipynb\n",
      "Converting 05_anneal.ipynb\n",
      "Converting 09_optimizer.ipynb\n",
      "Converting 04_callbacks.ipynb\n",
      "Converting 08_data_block.ipynb\n",
      "Converting 08a_heterogeneous_dictionary.ipynb\n",
      "Converting 06_cuda.ipynb\n",
      "Converting 01_matmul.ipynb\n",
      "Converting 02_fully_connected.ipynb\n",
      "Converting 01a_fastai_layers.ipynb\n",
      "Converting 11_imagenette.ipynb\n",
      "Converting 05b_early_stopping.ipynb\n",
      "Converting 00_load_data.ipynb\n",
      "Converting 03_minibatch_training.ipynb\n",
      "Converting 07_batchnorm.ipynb\n"
     ]
    }
   ],
   "source": [
    "makeLibrary(nbFolder, dest:dest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Script to notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "let fname = dest/\"LoadData.swift\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func readScript(_ fname: Path) -> String {\n",
    "    let data = try! Data(contentsOf: fname.url)\n",
    "    return String(data: data, encoding: .utf8)!\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func writeNotebook(_ nbFname: Path, nbData: [String: Any]) {\n",
    "    let outData = try! JSONSerialization.data(withJSONObject: nbData, options: .prettyPrinted)  \n",
    "    let jsonString = String(data: outData, encoding: .utf8)! \n",
    "    do { try jsonString.write(to: nbFname, encoding: .utf8) }\n",
    "    catch { \"Couldn't save notebook\" }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func scriptToNotebook(_ fname: Path, nbFolder: Path){\n",
    "    let code = readScript(fname)\n",
    "    let codeCells = code.components(separatedBy: \"//cell\")\n",
    "    let nbName = codeCells[0].components(separatedBy: \"\\n\")[1].components(separatedBy: \"from \")[1]\n",
    "    let nbFname = nbFolder / nbName\n",
    "    var jsonData = readNb(nbFname)\n",
    "    var cells = jsonData[\"cells\"] as! [[String:Any]]\n",
    "    for c in codeCells[1...] {\n",
    "        var lines = c.components(separatedBy: \"\\n\")\n",
    "        let idx: Int = Int(lines[0])!\n",
    "        var i = lines.count-1\n",
    "        while lines[i].isEmpty { i -= 1}\n",
    "        if i > 1 {\n",
    "            for i in 1...(i-1) { lines[i].append(\"\\n\") }\n",
    "        }\n",
    "        lines[0] = \"// export\\n\"\n",
    "        cells[idx][\"source\"] = Array(lines[...i])\n",
    "    }\n",
    "    jsonData[\"cells\"] = cells\n",
    "    writeNotebook(nbFname, nbData: jsonData)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "scriptToNotebook(fname, nbFolder: nbFolder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func updateNotebooks(_ scriptsFolder: Path, nbFolder: Path) {\n",
    "    for entry in try! scriptsFolder.ls()  where entry.kind == Entry.Kind.file && entry.path.basename().hasMatch(pat: #\".swift$\"#) {\n",
    "        print(\"Updating nb from \\(entry.path.basename())\")\n",
    "        scriptToNotebook(entry.path, nbFolder: nbFolder)\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating nb from WhySqrt5.swift\n",
      "Updating nb from Anneal.swift\n",
      "Updating nb from Callbacks.swift\n",
      "Updating nb from Optimizer.swift\n",
      "Updating nb from MixupLs.swift\n",
      "Updating nb from DataBlock.swift\n",
      "Updating nb from FastaiLayers.swift\n",
      "Updating nb from FullyConnected.swift\n",
      "Updating nb from HeterogeneousDictionary.swift\n",
      "Updating nb from Batchnorm.swift\n",
      "Updating nb from MinibatchTraining.swift\n",
      "Updating nb from LoadData.swift\n",
      "Updating nb from Matmul.swift\n",
      "Updating nb from Imagenette.swift\n",
      "Updating nb from EarlyStopping.swift\n",
      "Updating nb from Cuda.swift\n"
     ]
    }
   ],
   "source": [
    "updateNotebooks(dest, nbFolder: nbFolder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Diffing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "public func diffNbScript(_ nbFname: Path, dest: Path){\n",
    "    let newName = nbNameToScriptName(fname.basename(dropExtension: true))+\".swift\"\n",
    "    let destFname = (dest ?? fname.parent) / newName\n",
    "    let cells = readNb(nbFname)[\"cells\"] as! [[String:Any]]\n",
    "    \n",
    "    let data = try! Data(contentsOf: fname.url)\n",
    "    let code: String = String(data: data, encoding: .utf8)!\n",
    "    let codeCells = code.components(separatedBy: \"//cell\")\n",
    "    let nbName = codeCells[0].components(separatedBy: \"\\n\")[1].components(separatedBy: \"from \")[1]\n",
    "    let nbData = try! Data(contentsOf: nbFname.url)\n",
    "    var jsonData = try! JSONSerialization.jsonObject(with: nbData, options: .allowFragments) as! [String: Any]\n",
    "    var cells = jsonData[\"cells\"] as! [[String:Any]]\n",
    "    for c in codeCells[1...] {\n",
    "        var lines = c.components(separatedBy: \"\\n\")\n",
    "        let idx: Int = Int(lines[0])!\n",
    "        var i = lines.count-1\n",
    "        while lines[i].isEmpty { i -= 1}\n",
    "        if i > 1 {\n",
    "            for i in 1...(i-1) { lines[i].append(\"\\n\") }\n",
    "        }\n",
    "        lines[0] = \"// export\\n\"\n",
    "        cells[idx][\"source\"] = Array(lines[...i])\n",
    "    }\n",
    "    jsonData[\"cells\"] = cells\n",
    "    let outData = try! JSONSerialization.data(withJSONObject: jsonData, options: .prettyPrinted)  \n",
    "    let jsonString = String(data: outData, encoding: .utf8)! \n",
    "    do { try jsonString.write(to: nbFname, encoding: .utf8) }\n",
    "    catch { \"Couldn't save notebook\" }\n",
    "}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Swift",
   "language": "swift",
   "name": "swift"
  },
  "language_info": {
   "file_extension": ".swift",
   "mimetype": "text/x-swift",
   "name": "swift",
   "version": ""
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}