{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Behavioural Driven Development Testing for Jupyter Notebooks\n",
"\n",
"Handy way to process the run unit tests (via doctest) and integration tests (via behave) in jupyter notebooks (.ipynb) containing Python functions.\n",
"The script will convert an .ipynb to a string format (basically a .py file), loads them as modules, and runs the tests on them.\n",
"To run it in the console, do:\n",
"\n",
" python -m pytest --verbose --disable-warnings --nbval test_ipynb.ipynb\n",
"\n",
"The script should tell you which ipynb file's doctests has failed (e.g. srgan_train.ipynb).\n",
"You can then open up this very jupyter notebook to debug and inspect the situation further."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from features.environment import _load_ipynb_modules\n",
"import behave.__main__\n",
"\n",
"import doctest\n",
"import os\n",
"import sys\n",
"\n",
"\n",
"def _unit_test_ipynb(path: str):\n",
" \"\"\"\n",
" Unit tests on loaded modules from a .ipynb file.\n",
" Uses doctest.\n",
" \"\"\"\n",
" assert path.endswith(\".ipynb\")\n",
"\n",
" module = _load_ipynb_modules(ipynb_path=path)\n",
" num_failures, num_attempted = doctest.testmod(m=module, verbose=True)\n",
" if num_failures > 0:\n",
" sys.exit(num_failures)\n",
"\n",
"\n",
"def _integration_test_ipynb(path: str, summary: bool = False):\n",
" \"\"\"\n",
" Integration tests on various feature behaviours inside a .feature file.\n",
" Uses behave.\n",
" \"\"\"\n",
" assert os.path.exists(path=path)\n",
" assert path.endswith(\".feature\")\n",
"\n",
" if summary == False:\n",
" args = f\"--tags ~@skip --no-summary {path}\"\n",
" elif summary == True:\n",
" args = f\"--tags ~@skip {path}\"\n",
"\n",
" num_failures = behave.__main__.main(args=args)\n",
" if num_failures > 0:\n",
" sys.exit(num_failures)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Unit tests\n",
"Uses [doctest](https://en.wikipedia.org/wiki/Doctest).\n",
"Small tests for each individual function."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Trying:\n",
" os.makedirs(name=\"/tmp/highres\", exist_ok=True)\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" download_to_path(path=\"/tmp/highres/2011_Antarctica_TO.csv\",\n",
" url=\"https://data.cresis.ku.edu/data/rds/2011_Antarctica_TO/csv_good/2011_Antarctica_TO.csv\")\n",
"Expecting:\n",
" <Response [200]>\n",
"ok\n",
"Trying:\n",
" _ = shutil.copy(src=\"highres/20xx_Antarctica_TO.json\", dst=\"/tmp/highres\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" df = ascii_to_xyz(pipeline_file=\"/tmp/highres/20xx_Antarctica_TO.json\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" df.head(2)\n",
"Expecting:\n",
" x y z\n",
" 0 345580.826265 -1.156471e+06 -377.2340\n",
" 1 345593.322948 -1.156460e+06 -376.6332\n",
"ok\n",
"Trying:\n",
" shutil.rmtree(path=\"/tmp/highres\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" download_to_path(path=\"highres/Data_20171204_02.csv\",\n",
" url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/Data_20171204_02.csv\")\n",
"Expecting:\n",
" <Response [200]>\n",
"ok\n",
"Trying:\n",
" check_sha256(\"highres/Data_20171204_02.csv\")\n",
"Expecting:\n",
" '53cef7a0d28ff92b30367514f27e888efbc32b1bda929981b371d2e00d4c671b'\n",
"ok\n",
"Trying:\n",
" os.remove(path=\"highres/Data_20171204_02.csv\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" download_to_path(path=\"highres/Data_20171204_02.csv\",\n",
" url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/Data_20171204_02.csv\")\n",
"Expecting:\n",
" <Response [200]>\n",
"ok\n",
"Trying:\n",
" open(\"highres/Data_20171204_02.csv\").readlines()\n",
"Expecting:\n",
" ['LAT,LON,UTCTIMESOD,THICK,ELEVATION,FRAME,SURFACE,BOTTOM,QUALITY\\n']\n",
"ok\n",
"Trying:\n",
" os.remove(path=\"highres/Data_20171204_02.csv\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" xyz_data = pd.DataFrame(np.random.RandomState(seed=42).rand(30).reshape(10, 3))\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" get_region(xyz_data=xyz_data)\n",
"Expecting:\n",
" '0.05808/0.83244/0.02058/0.95071'\n",
"ok\n",
"Trying:\n",
" xr.DataArray(\n",
" data=np.zeros(shape=(36, 32)),\n",
" coords={\"x\": np.arange(1, 37), \"y\": np.arange(1, 33)},\n",
" dims=[\"x\", \"y\"],\n",
" ).to_netcdf(path=\"/tmp/tmp_wb.nc\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" get_window_bounds(filepath=\"/tmp/tmp_wb.nc\")\n",
"Expecting:\n",
" Tiling: /tmp/tmp_wb.nc ... 2\n",
" [(0.5, 4.5, 32.5, 36.5), (0.5, 0.5, 32.5, 32.5)]\n",
"ok\n",
"Trying:\n",
" os.remove(\"/tmp/tmp_wb.nc\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" xr.DataArray(\n",
" data=np.random.RandomState(seed=42).rand(64).reshape(8, 8),\n",
" coords={\"x\": np.arange(8), \"y\": np.arange(8)},\n",
" dims=[\"x\", \"y\"],\n",
" ).to_netcdf(path=\"/tmp/tmp_st.nc\", mode=\"w\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" selective_tile(\n",
" filepath=\"/tmp/tmp_st.nc\",\n",
" window_bounds=[(1.0, 4.0, 3.0, 6.0), (2.0, 5.0, 4.0, 7.0)],\n",
" )\n",
"Expecting:\n",
" Tiling: /tmp/tmp_st.nc\n",
" array([[[[0.18485446, 0.96958464],\n",
" [0.4951769 , 0.03438852]]],\n",
" <BLANKLINE>\n",
" <BLANKLINE>\n",
" [[[0.04522729, 0.32533032],\n",
" [0.96958464, 0.77513283]]]], dtype=float32)\n",
"ok\n",
"Trying:\n",
" os.remove(\"/tmp/tmp_st.nc\")\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" xyz_data = 1000*pd.DataFrame(np.random.RandomState(seed=42).rand(60).reshape(20, 3))\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" region = get_region(xyz_data=xyz_data)\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" grid = xyz_to_grid(xyz_data=xyz_data, region=region, spacing=250)\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" grid.to_array().shape\n",
"Expecting:\n",
" (1, 5, 5)\n",
"ok\n",
"Trying:\n",
" grid.to_array().values\n",
"Expecting:\n",
" array([[[403.17618 , 544.92535 , 670.7824 , 980.75055 , 961.47723 ],\n",
" [379.0757 , 459.26407 , 314.38297 , 377.78555 , 546.0469 ],\n",
" [450.67664 , 343.26 , 88.391594, 260.10492 , 452.3337 ],\n",
" [586.09906 , 469.74008 , 216.8168 , 486.9802 , 642.2116 ],\n",
" [451.4794 , 652.7244 , 325.77896 , 879.8973 , 916.7921 ]]],\n",
" dtype=float32)\n",
"ok\n",
"2 items had no tests:\n",
" data_prep\n",
" data_prep.parse_datalist\n",
"7 items passed all tests:\n",
" 6 tests in data_prep.ascii_to_xyz\n",
" 3 tests in data_prep.check_sha256\n",
" 3 tests in data_prep.download_to_path\n",
" 2 tests in data_prep.get_region\n",
" 3 tests in data_prep.get_window_bounds\n",
" 3 tests in data_prep.selective_tile\n",
" 5 tests in data_prep.xyz_to_grid\n",
"25 tests in 9 items.\n",
"25 passed and 0 failed.\n",
"Test passed.\n"
]
}
],
"source": [
"_unit_test_ipynb(path=\"data_prep.ipynb\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Trying:\n",
" discriminator_model = DiscriminatorModel()\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" y_pred = discriminator_model.forward(\n",
" x=np.random.rand(2, 1, 32, 32).astype(\"float32\")\n",
" )\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" y_pred.shape\n",
"Expecting:\n",
" (2, 1)\n",
"ok\n",
"Trying:\n",
" discriminator_model.count_params()\n",
"Expecting:\n",
" 10205129\n",
"ok\n",
"Trying:\n",
" generator_model = GeneratorModel()\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" y_pred = generator_model.forward(\n",
" x=np.random.rand(1, 1, 10, 10).astype(\"float32\"),\n",
" w1=np.random.rand(1, 1, 100, 100).astype(\"float32\"),\n",
" w2=np.random.rand(1, 1, 20, 20).astype(\"float32\"),\n",
" )\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" y_pred.shape\n",
"Expecting:\n",
" (1, 1, 32, 32)\n",
"ok\n",
"Trying:\n",
" generator_model.count_params()\n",
"Expecting:\n",
" 9088641\n",
"ok\n",
"Trying:\n",
" calculate_discriminator_loss(\n",
" real_labels_pred=chainer.variable.Variable(data=np.array([[1.1], [-0.5]])),\n",
" fake_labels_pred=chainer.variable.Variable(data=np.array([[-0.3], [1.0]])),\n",
" real_minus_fake_target=np.array([[1], [1]]),\n",
" fake_minus_real_target=np.array([[0], [0]]),\n",
" )\n",
"Expecting:\n",
" variable(1.56670504)\n",
"ok\n",
"Trying:\n",
" calculate_generator_loss(\n",
" y_pred=chainer.variable.Variable(data=np.ones(shape=(2, 1, 3, 3))),\n",
" y_true=np.full(shape=(2, 1, 3, 3), fill_value=10.0),\n",
" fake_labels=np.array([[-1.2], [0.5]]),\n",
" real_labels=np.array([[0.5], [-0.8]]),\n",
" fake_minus_real_target=np.array([[1], [1]]).astype(np.int32),\n",
" real_minus_fake_target=np.array([[0], [0]]).astype(np.int32),\n",
" )\n",
"Expecting:\n",
" variable(0.09867307)\n",
"ok\n",
"Trying:\n",
" psnr(\n",
" y_true=np.ones(shape=(2, 1, 3, 3)),\n",
" y_pred=np.full(shape=(2, 1, 3, 3), fill_value=2),\n",
" )\n",
"Expecting:\n",
" 192.65919722494797\n",
"ok\n",
"Trying:\n",
" model = GeneratorModel()\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" _, _ = save_model_weights_and_architecture(\n",
" trained_model=model, save_path=\"/tmp/weights\"\n",
" )\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" os.path.exists(path=\"/tmp/weights/srgan_generator_model_architecture.onnx.txt\")\n",
"Expecting:\n",
" True\n",
"ok\n",
"Trying:\n",
" train_arrays = {\n",
" \"X\": np.random.RandomState(seed=42).rand(2, 1, 10, 10).astype(np.float32),\n",
" \"W1\": np.random.RandomState(seed=42).rand(2, 1, 100, 100).astype(np.float32),\n",
" \"W2\": np.random.RandomState(seed=42).rand(2, 1, 20, 20).astype(np.float32),\n",
" \"Y\": np.random.RandomState(seed=42).rand(2, 1, 32, 32).astype(np.float32),\n",
" }\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" discriminator_model = DiscriminatorModel()\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" discriminator_optimizer = chainer.optimizers.Adam(alpha=0.001, eps=1e-7).setup(\n",
" link=discriminator_model\n",
" )\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" generator_model = GeneratorModel()\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" d_weight0 = [d for d in discriminator_model.params()][-3][0].array\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" d_train_loss, d_train_accu = train_eval_discriminator(\n",
" input_arrays=train_arrays,\n",
" g_model=generator_model,\n",
" d_model=discriminator_model,\n",
" d_optimizer=discriminator_optimizer,\n",
" )\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" d_weight1 = [d for d in discriminator_model.params()][-3][0].array\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" d_weight0 != d_weight1 #check that training has occurred (i.e. weights changed)\n",
"Expecting:\n",
" True\n",
"ok\n",
"Trying:\n",
" train_arrays = {\n",
" \"X\": np.random.RandomState(seed=42).rand(2, 1, 10, 10).astype(np.float32),\n",
" \"W1\": np.random.RandomState(seed=42).rand(2, 1, 100, 100).astype(np.float32),\n",
" \"W2\": np.random.RandomState(seed=42).rand(2, 1, 20, 20).astype(np.float32),\n",
" \"Y\": np.random.RandomState(seed=42).rand(2, 1, 32, 32).astype(np.float32),\n",
" }\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" generator_model = GeneratorModel()\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" generator_optimizer = chainer.optimizers.Adam(alpha=0.001, eps=1e-7).setup(\n",
" link=generator_model\n",
" )\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" discriminator_model = DiscriminatorModel()\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" g_weight0 = [g for g in generator_model.params()][8][0, 0, 0, 0].array\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" _ = train_eval_generator(\n",
" input_arrays=train_arrays,\n",
" g_model=generator_model,\n",
" d_model=discriminator_model,\n",
" g_optimizer=generator_optimizer,\n",
" )\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" g_weight1 = [g for g in generator_model.params()][8][0, 0, 0, 0].array\n",
"Expecting nothing\n",
"ok\n",
"Trying:\n",
" g_weight0 != g_weight1 #check that training has occurred (i.e. weights changed)\n",
"Expecting:\n",
" True\n",
"ok\n",
"20 items had no tests:\n",
" srgan_train\n",
" srgan_train.DeepbedmapInputBlock\n",
" srgan_train.DeepbedmapInputBlock.__init__\n",
" srgan_train.DeepbedmapInputBlock.forward\n",
" srgan_train.DiscriminatorModel.__init__\n",
" srgan_train.DiscriminatorModel.forward\n",
" srgan_train.GeneratorModel.__init__\n",
" srgan_train.GeneratorModel.forward\n",
" srgan_train.ResInResDenseBlock\n",
" srgan_train.ResInResDenseBlock.__init__\n",
" srgan_train.ResInResDenseBlock.forward\n",
" srgan_train.ResidualDenseBlock\n",
" srgan_train.ResidualDenseBlock.__init__\n",
" srgan_train.ResidualDenseBlock.forward\n",
" srgan_train.compile_srgan_model\n",
" srgan_train.get_deepbedmap_test_result\n",
" srgan_train.get_train_dev_iterators\n",
" srgan_train.load_data_into_memory\n",
" srgan_train.objective\n",
" srgan_train.trainer\n",
"8 items passed all tests:\n",
" 4 tests in srgan_train.DiscriminatorModel\n",
" 4 tests in srgan_train.GeneratorModel\n",
" 1 tests in srgan_train.calculate_discriminator_loss\n",
" 1 tests in srgan_train.calculate_generator_loss\n",
" 1 tests in srgan_train.psnr\n",
" 3 tests in srgan_train.save_model_weights_and_architecture\n",
" 8 tests in srgan_train.train_eval_discriminator\n",
" 8 tests in srgan_train.train_eval_generator\n",
"30 tests in 28 items.\n",
"30 passed and 0 failed.\n",
"Test passed.\n"
]
}
],
"source": [
"_unit_test_ipynb(path=\"srgan_train.ipynb\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Integration tests\n",
"\n",
"Uses [behave](https://github.com/behave/behave).\n",
"Medium sized tests which checks that components work together properly.\n",
"Ensures that the behaviour of features (made up of units) is sound."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"@fixture.data_prep\n",
"Feature: Data preparation # features/data_prep.feature:3\n",
" In order to have reproducible data inputs for everyone\n",
" As a data scientist,\n",
" We want to share cryptographically secured pieces of the datasets\n",
" Scenario Outline: Download and check data -- @1.1 Files to download and check # features/data_prep.feature:15\n",
" Given this https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/Data_20171204_02.csv link to a file hosted on the web # features/steps/test_data_prep.py:8\n",
" When we download it to highres/Data_20171204_02.csv # features/steps/test_data_prep.py:13\n",
" Then the local file should have this 53cef7a0d28ff92b30367514f27e888efbc32b1bda929981b371d2e00d4c671b checksum # features/steps/test_data_prep.py:19\n",
"\n",
" Scenario Outline: Download and check data -- @1.2 Files to download and check # features/data_prep.feature:16\n",
" Given this http://ramadda.nerc-bas.ac.uk/repository/entry/get/Polar%20Data%20Centre/DOI/Rutford%20Ice%20Stream%20bed%20elevation%20DEM%20from%20radar%20data/bed_WGS84_grid.txt?entryid=synth%3A54757cbe-0b13-4385-8b31-4dfaa1dab55e%3AL2JlZF9XR1M4NF9ncmlkLnR4dA%3D%3D link to a file hosted on the web # features/steps/test_data_prep.py:8\n",
" When we download it to highres/bed_WGS84_grid.txt # features/steps/test_data_prep.py:13\n",
" Then the local file should have this 7396e56cda5adb82cecb01f0b3e01294ed0aa6489a9629f3f7e8858ea6cb91cf checksum # features/steps/test_data_prep.py:19\n",
"\n",
" Scenario Outline: Grid datasets -- @1.1 ASCII text files to grid # features/data_prep.feature:26\n",
" Given a collection of raw high resolution datasets bed_WGS84_grid.txt # features/steps/test_data_prep.py:25\n",
" When we process the data through bed_WGS84_grid.json # features/steps/test_data_prep.py:38\n",
" And interpolate the xyz data table to bed_WGS84_grid.nc # features/steps/test_data_prep.py:45\n",
" Then a high resolution raster grid is returned # features/steps/test_data_prep.py:54\n",
"\n",
" Scenario Outline: Tile datasets -- @1.1 Raster grids to tile # features/data_prep.feature:36\n",
" Given a big highres raster grid 2010tr.nc # features/steps/test_data_prep.py:60\n",
" And a collection of square bounding boxes \"model/train/tiles_3031.geojson\" # features/steps/test_data_prep.py:70\n",
" When we crop the big raster grid using those bounding boxes # features/steps/test_data_prep.py:80\n",
" Then a stack of small raster tiles is returned # features/steps/test_data_prep.py:87\n",
"\n"
]
}
],
"source": [
"_integration_test_ipynb(path=\"features/data_prep.feature\")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"@fixture.srgan_train\n",
"Feature: Train Super Resolution Model # features/srgan_train.feature:3\n",
" In order to have a well performing super resolution model\n",
" As a machine learning engineer,\n",
" We want to craft and teach the model to do well on a test area\n",
" Background: Load the prepared data # features/srgan_train.feature:8\n",
"\n",
" Scenario Outline: Train Super Resolution Model with fixed hyperparameters -- @1.1 Fixed hyperparameters # features/srgan_train.feature:19\n",
" Given a prepared collection of tiled raster data # features/steps/test_srgan_train.py:6\n",
" Given some hyperparameter settings 1 0.3 5e-4 # features/steps/test_srgan_train.py:14\n",
" And a compiled neural network model # features/steps/test_srgan_train.py:25\n",
" When the model is trained for a while # features/steps/test_srgan_train.py:35\n",
" Then we know how well the model performs on our test area # features/steps/test_srgan_train.py:60\n",
"\n"
]
}
],
"source": [
"_integration_test_ipynb(path=\"features/srgan_train.feature\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"@fixture.deepbedmap\n",
"Feature: DeepBedMap # features/deepbedmap.feature:3\n",
" In order to create a great map of Antarctica's bed\n",
" As a scientist,\n",
" We want a model that produces realistic images from many open datasets\n",
" Scenario Outline: Determine high resolution bed -- @1.1 Bounding box views of Antarctica # features/deepbedmap.feature:16\n",
" Given some view of Antarctica -1593714.328,-164173.7848,-1575464.328,-97923.7848 # features/steps/test_deepbedmap.py:6\n",
" When we gather low and high resolution images related to that view # features/steps/test_deepbedmap.py:14\n",
" And pass those images into our trained neural network model # features/steps/test_deepbedmap.py:30\n",
" Then a four times upsampled super resolution bed elevation map is returned # features/steps/test_deepbedmap.py:38\n",
"\n"
]
}
],
"source": [
"_integration_test_ipynb(path=\"features/deepbedmap.feature\")"
]
}
],
"metadata": {
"jupytext": {
"formats": "ipynb,py:percent"
},
"kernelspec": {
"display_name": "deepbedmap",
"language": "python",
"name": "deepbedmap"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}