{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|hide\n", "#|default_exp maker" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "from __future__ import annotations" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# maker\n", "> Create one or more modules from selected notebook cells" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "from nbdev.read import *\n", "from nbdev.imports import *\n", "\n", "from fastcore.script import *\n", "from fastcore.basics import *\n", "from fastcore.imports import *\n", "from execnb.nbio import *\n", "\n", "import ast,contextlib\n", "\n", "from collections import defaultdict\n", "from pprint import pformat\n", "from textwrap import TextWrapper" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|hide\n", "from fastcore.test import *\n", "from pdb import set_trace\n", "from importlib import reload" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Variable helpers" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "These functions let us find and modify the definitions of variables in Python modules." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def find_var(lines, varname):\n", " \"Find the line numbers where `varname` is defined in `lines`\"\n", " start = first(i for i,o in enumerate(lines) if o.startswith(varname))\n", " if start is None: return None,None\n", " empty = ' ','\\t'\n", " if start==len(lines)-1 or lines[start+1][:1] not in empty: return start,start+1\n", " end = first(i for i,o in enumerate(lines[start+1:]) if o[:1] not in empty)\n", " return start,len(lines) if end is None else (end+start+1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "t = '''a_=(1,\n", " 2,\n", " 3)\n", "\n", "b_=3'''\n", "test_eq(find_var(t.splitlines(), 'a_'), (0,3))\n", "test_eq(find_var(t.splitlines(), 'b_'), (4,5))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def read_var(code, varname):\n", " \"Eval and return the value of `varname` defined in `code`\"\n", " lines = code.splitlines()\n", " start,end = find_var(lines, varname)\n", " if start is None: return None\n", " res = [lines[start].split('=')[-1].strip()]\n", " res += lines[start+1:end]\n", " try: return eval('\\n'.join(res))\n", " except SyntaxError: raise Exception('\\n'.join(res)) from None" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_eq(read_var(t, 'a_'), (1,2,3))\n", "test_eq(read_var(t, 'b_'), 3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def update_var(varname, func, fn=None, code=None):\n", " \"Update the definition of `varname` in file `fn`, by calling `func` with the current definition\"\n", " if fn:\n", " fn = Path(fn)\n", " code = fn.read_text()\n", " lines = code.splitlines()\n", " v = read_var(code, varname)\n", " res = func(v)\n", " start,end = find_var(lines, varname)\n", " del(lines[start:end])\n", " lines.insert(start, f\"{varname} = {res}\")\n", " code = '\\n'.join(lines)\n", " if fn: fn.write_text(code)\n", " else: return code" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "g = exec_new(t)\n", "test_eq((g['a_'],g['b_']), ((1,2,3),3))\n", "t2 = update_var('a_', lambda o:0, code=t)\n", "exec(t2, g)\n", "test_eq((g['a_'],g['b_']), (0,3))\n", "t3 = update_var('b_', lambda o:0, code=t)\n", "exec(t3, g)\n", "test_eq((g['a_'],g['b_']), ((1,2,3),0))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## ModuleMaker -" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "class ModuleMaker:\n", " \"Helper class to create exported library from notebook source cells\"\n", " def __init__(self, dest, name, nb_path, is_new=True, parse=True):\n", " dest,nb_path = Path(dest),Path(nb_path)\n", " store_attr()\n", " self.fname = dest/(name.replace('.','/') + \".py\")\n", " if is_new: dest.mkdir(parents=True, exist_ok=True)\n", " else: assert self.fname.exists(), f\"{self.fname} does not exist\"\n", " self.dest2nb = nb_path.relpath(dest)\n", " self.hdr = f\"# %% {self.dest2nb}\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In order to export a notebook, we need an way to create a Python file. `ModuleMaker` fills that role. Pass in the directory where you want to module created, the name of the module, the path of the notebook source, and set `is_new` to `True` if this is a new file being created (rather than an existing file being added to). The location of the saved module will be in `fname`. Finally, if the source in the notebooks should not be parsed by Python (such as partial class declarations in cells), `parse` should be set to `False`.\n", "\n", "> Note: If doing so, then the `__all__` generation will be turned off as well." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Path('tmp/test/testing.py')" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mm = ModuleMaker(dest='tmp', name='test.testing', nb_path=Path.cwd()/'01_export.ipynb', is_new=True)\n", "mm.fname" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def decor_id(d):\n", " \"`id` attr of decorator, regardless of whether called as function or bare\"\n", " return d.id if hasattr(d, 'id') else nested_attr(d, 'func.id', '')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "_def_types = ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef\n", "_assign_types = ast.AnnAssign, ast.Assign, ast.AugAssign\n", "\n", "def _val_or_id(it): \n", " if sys.version_info < (3,8): return [getattr(o, 's', None) for o in it.value.elts]\n", " else:return [getattr(o, 'value', getattr(o, 'id', None)) for o in it.value.elts]\n", "def _all_targets(a): return L(getattr(a,'elts',a))\n", "def _filt_dec(x): return decor_id(x).startswith('patch')\n", "def _wants(o): return isinstance(o,_def_types) and not any(L(o.decorator_list).filter(_filt_dec))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def retr_exports(trees):\n", " # include anything mentioned in \"_all_\", even if otherwise private\n", " # NB: \"_all_\" can include strings (names), or symbols, so we look for \"id\" or \"value\"\n", " assigns = trees.filter(risinstance(_assign_types))\n", " all_assigns = assigns.filter(lambda o: getattr(o.targets[0],'id',None)=='_all_')\n", " all_vals = all_assigns.map(_val_or_id).concat()\n", " syms = trees.filter(_wants).attrgot('name')\n", "\n", " # assignment targets (NB: can be multiple, e.g. \"a=b=c\", and/or destructuring e.g \"a,b=(1,2)\")\n", " assign_targs = L(L(assn.targets).map(_all_targets).concat() for assn in assigns).concat()\n", " exports = (assign_targs.attrgot('id')+syms).filter(lambda o: o and o[0]!='_')\n", " return (exports+all_vals).unique()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "@patch\n", "def make_all(self:ModuleMaker, cells):\n", " \"Create `__all__` with all exports in `cells`\"\n", " if cells is None: return ''\n", " return retr_exports(cells.map(NbCell.parsed_).concat())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def make_code_cell(code): return AttrDict(source=code, cell_type=\"code\", execution_count=None)\n", "def make_code_cells(*ss): return dict2nb({'cells':L(ss).map(make_code_cell)}).cells" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We want to add an `__all__` to the top of the exported module. This methods autogenerates it from all code in `cells`." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "nb = make_code_cells(\"from __future__ import print_function\", \"def a():...\", \"def b():...\",\n", " \"c=d=1\", \"_f=1\", \"_g=1\", \"_all_=['_g']\", \"@patch\\ndef h(self:ca):...\")\n", "test_eq(set(mm.make_all(nb)), set(['a','b','c','d', '_g']))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def relative_import(name, fname, level=0):\n", " \"Convert a module `name` to a name relative to `fname`\"\n", " assert not level\n", " sname = name.replace('.','/')\n", " if not(os.path.commonpath([sname,fname])): return name\n", " rel = os.path.relpath(sname, fname)\n", " if rel==\".\": return \".\"\n", " res = rel.replace(f\"..{os.path.sep}\", \".\")\n", " return \".\" + res.replace(os.path.sep, \".\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_eq(relative_import('nbdev.core', \"xyz\"), 'nbdev.core')\n", "test_eq(relative_import('nbdev.core', 'nbdev'), '.core')\n", "_p = Path('fastai')\n", "test_eq(relative_import('fastai.core', _p/'vision'), '..core')\n", "test_eq(relative_import('fastai.core', _p/'vision/transform'), '...core')\n", "test_eq(relative_import('fastai.vision.transform', _p/'vision'), '.transform')\n", "test_eq(relative_import('fastai.notebook.core', _p/'data'), '..notebook.core')\n", "test_eq(relative_import('fastai.vision', _p/'vision'), '.')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "# Based on https://github.com/thonny/thonny/blob/master/thonny/ast_utils.py\n", "def _mark_text_ranges(\n", " source: str|bytes, # Source code to add ranges to\n", "):\n", " \"Adds `end_lineno` and `end_col_offset` to each `node` recursively. Used for Python 3.7 compatibility\"\n", " from asttokens.asttokens import ASTTokens\n", " # We need to reparse the source to get a full tree to walk\n", " root = ast.parse(source)\n", " ASTTokens(source, tree=root)\n", " for child in ast.walk(root):\n", " if hasattr(child,\"last_token\"):\n", " child.end_lineno,child.end_col_offset = child.last_token.end\n", " # Some tokens stay without end info\n", " if hasattr(child,\"lineno\") and (not hasattrs(child, [\"end_lineno\",\"end_col_offset\"])):\n", " child.end_lineno, child.end_col_offset = child.lineno, child.col_offset+2\n", " return root.body" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def update_import(source, tree, libname, f=relative_import):\n", " if not tree: return\n", " if sys.version_info < (3,8): tree = _mark_text_ranges(source)\n", " imps = L(tree).filter(risinstance(ast.ImportFrom))\n", " if not imps: return\n", " src = source.splitlines(True)\n", " for imp in imps:\n", " nmod = f(imp.module, libname, imp.level)\n", " lin = imp.lineno-1\n", " sec = src[lin][imp.col_offset:imp.end_col_offset]\n", " newsec = re.sub(f\"(from +){'.'*imp.level}{imp.module}\", fr\"\\1{nmod}\", sec)\n", " src[lin] = src[lin].replace(sec,newsec)\n", " return src\n", "\n", "@patch\n", "def import2relative(cell:NbCell, libname):\n", " src = update_import(cell.source, cell.parsed_(), libname)\n", " if src: cell.set_source(src)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ss = \"from nbdev.export import *\\nfrom nbdev.a.b import *\"\n", "cell = make_code_cells([ss])[0]\n", "cell.import2relative('nbdev')\n", "test_eq(cell.source, 'from .export import *\\nfrom .a.b import *')\n", "\n", "cell = make_code_cells([ss])[0]\n", "cell.import2relative('nbdev/a')\n", "test_eq(cell.source, 'from ..export import *\\nfrom .b import *')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "@patch\n", "def _last_future(self:ModuleMaker, cells):\n", " \"Returns the location of a `__future__` in `cells`\"\n", " trees = cells.map(NbCell.parsed_)\n", " try: return max(i for i,tree in enumerate(trees) if tree and any(\n", " isinstance(t,ast.ImportFrom) and t.module=='__future__' for t in tree))+1\n", " except ValueError: return 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def _import2relative(cells, lib_name=None):\n", " \"Converts `cells` to use `import2relative` based on `lib_name`\"\n", " if lib_name is None: lib_name = get_config().lib_name\n", " for cell in cells: cell.import2relative(lib_name)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "@patch\n", "def make(self:ModuleMaker, cells, all_cells=None, lib_name=None):\n", " \"Write module containing `cells` with `__all__` generated from `all_cells`\"\n", " if all_cells is None: all_cells = cells\n", " if self.parse: \n", " libnm = get_config().path('lib_path')\n", " mod_dir = os.path.relpath(self.fname.parent, libnm.parent)\n", " _import2relative(all_cells, mod_dir)\n", " if not self.is_new: return self._make_exists(cells, all_cells)\n", "\n", " self.fname.parent.mkdir(exist_ok=True, parents=True)\n", " last_future = 0\n", " if self.parse:\n", " _all = self.make_all(all_cells)\n", " last_future = self._last_future(cells) if len(all_cells)>0 else 0\n", " tw = TextWrapper(width=120, initial_indent='', subsequent_indent=' '*11, break_long_words=False)\n", " all_str = '\\n'.join(tw.wrap(str(_all)))\n", " with self.fname.open('w') as f:\n", " f.write(f\"# AUTOGENERATED! DO NOT EDIT! File to edit: {self.dest2nb}.\")\n", " if last_future > 0: write_cells(cells[:last_future], self.hdr, f)\n", " if self.parse: f.write(f\"\\n\\n# %% auto 0\\n__all__ = {all_str}\")\n", " write_cells(cells[last_future:], self.hdr, f, 1 if last_future>0 else 0)\n", " f.write('\\n')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def _print_file(fname, mx=None): print(Path(fname).read_text().strip()[:ifnone(mx,9999)])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "# AUTOGENERATED! DO NOT EDIT! File to edit: ../01_export.ipynb.\n", "\n", "# %% ../01_export.ipynb 0\n", "from __future__ import print_function\n", "\n", "# %% auto 0\n", "__all__ = ['a']\n", "\n", "# %% ../01_export.ipynb 2\n", "#|export\n", "def a(): ...\n", "\n", "# %% ../01_export.ipynb 3\n", "def b(): ...\n", "\n" ] } ], "source": [ "cells = make_code_cells(\"from __future__ import print_function\", \"#|export\\ndef a(): ...\", \"def b(): ...\")\n", "mm.make(cells, L([cells[1]]))\n", "print(Path('tmp/test/testing.py').read_text())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Pass `all_cells=[]` or `parse=False` if you don't want any `__all__` added." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Passing `parse=False` is also handy for when writing broken up functions or classes that `ast.parse` might not like but still want it to be exported, such as having once cell with the contents of:\n", "```python\n", "#|export\n", "class A:\n", "```\n", "Note that by doing so we cannot properly generate a `__all__`, so we assume that it is unwanted. " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Path('tmp/test/testing_noall.py')" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" } ], "source": [ "am = ModuleMaker(dest='tmp', name='test.testing_noall', nb_path=Path.cwd()/'01_export.ipynb', is_new=True, parse=False)\n", "am.fname" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "# AUTOGENERATED! DO NOT EDIT! File to edit: ../01_export.ipynb.\n", "\n", "# %% ../01_export.ipynb 0\n", "from __future__ import print_function\n", "\n", "# %% ../01_export.ipynb 1\n", "#|export\n", "def a(): ...\n", "\n", "# %% ../01_export.ipynb 2\n", "#|export\n", "class A:\n", "\n" ] } ], "source": [ "cells = make_code_cells(\"from __future__ import print_function\", \"#|export\\ndef a(): ...\", \"#|export\\nclass A:\")\n", "am.make(cells)\n", "print(Path('tmp/test/testing_noall.py').read_text())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "@patch\n", "def _update_all(self:ModuleMaker, all_cells, alls):\n", " return pformat(alls + self.make_all(all_cells), width=160)\n", "\n", "@patch\n", "def _make_exists(self:ModuleMaker, cells, all_cells=None):\n", " \"`make` for `is_new=False`\"\n", " if all_cells and self.parse: update_var('__all__', partial(self._update_all, all_cells), fn=self.fname)\n", " with self.fname.open('a') as f: write_cells(cells, self.hdr, f)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If `is_new=False` then the additional definitions are added to the bottom, and any existing `__all__` is updated with the newly-added symbols." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "c2 = make_code_cells(\"def c(): ...\", \"def d(): ...\")\n", "mm = ModuleMaker(dest='tmp', name='test.testing', nb_path=Path.cwd()/'01_export.ipynb', is_new=False)\n", "mm.make(c2, c2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "# AUTOGENERATED! DO NOT EDIT! File to edit: ../01_export.ipynb.\n", "\n", "# %% ../01_export.ipynb 0\n", "from __future__ import print_function\n", "\n", "# %% auto 0\n", "__all__ = ['a', 'c', 'd']\n", "\n", "# %% ../01_export.ipynb 2\n", "#|export\n", "def a(): ...\n", "\n", "# %% ../01_export.ipynb 3\n", "def b(): ...\n", "\n", "# %% ../01_export.ipynb 0\n", "def c(): ...\n", "\n", "# %% ../01_export.ipynb 1\n", "def d(): ...\n" ] } ], "source": [ "print(Path('tmp/test/testing.py').read_text())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "g = exec_import('.tmp.test.testing', '*')\n", "for s in \"a c d\".split(): assert s in g, s\n", "assert 'b' not in g\n", "assert g['a']() is None" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Export -" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|export\n", "def basic_export_nb2(fname, name, dest=None):\n", " \"A basic exporter to bootstrap nbdev using `ModuleMaker`\"\n", " if dest is None: dest = get_config().path('lib_path')\n", " cells = L(c for c in read_nb(fname).cells if re.match(r'#\\|\\s*export', c.source))\n", " ModuleMaker(dest=dest, name=name, nb_path=fname).make(cells)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#|eval: false\n", "path = Path('../nbdev')\n", "(path/'read.py').unlink(missing_ok=True)\n", "(path/'maker.py').unlink(missing_ok=True)\n", "\n", "add_init(path)\n", "cfg = get_config()\n", "\n", "basic_export_nb2('01_read.ipynb', 'read')\n", "basic_export_nb2('02_maker.ipynb', 'maker')\n", "\n", "g = exec_import('nbdev', 'maker')\n", "assert g['maker'].ModuleMaker\n", "assert 'ModuleMaker' in g['maker'].__all__" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 4 }