{ "cells": [ { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# Image Processing - Filtering\n", "\n", "Jan Eglinger\n", "\n", "![FMI](http://www.fmi.ch/img/logo-FMI-grey.gif)\n", "\n", "Facility for Advanced Imaging and Microscopy (FAIM)
\n", "Friedrich Miescher Institute for Biomedical Research (FMI)\n", "Basel, Switzerland\n", "\n", "Basel, March 7, 2018\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "init_cell": true, "slideshow": { "slide_type": "skip" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Added new repo: imagej.public\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "method": "display_data" }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "36c6c02e-2337-4b3d-8d2c-343d8ab79309", "version_major": 2, "version_minor": 0 }, "method": "display_data" }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Mar 12, 2019 4:08:17 PM java.util.prefs.WindowsPreferences \r\n", "WARNING: Could not open/create prefs root node Software\\JavaSoft\\Prefs at root 0x80000002. Windows RegCreateKeyEx(...) returned error code 5.\r\n" ] }, { "data": { "text/plain": [ "ImageJ initialized" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "//load ImageJ\n", "%classpath config resolver imagej.public https://maven.imagej.net/content/groups/public\n", "%classpath add mvn net.imagej imagej 2.0.0-rc-71\n", "\n", "//create ImageJ object\n", "ij = new net.imagej.ImageJ()\n", "\n", "notebook = ij.notebook()\n", "datasetIO = ij.scifio().datasetIO()\n", "ops = ij.op()\n", "\"ImageJ initialized\"" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "hide_input": false, "init_cell": true, "slideshow": { "slide_type": "skip" } }, "outputs": [ { "data": { "text/plain": [ "null" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "/* Required Imports */\n", "import net.imglib2.type.numeric.real.FloatType\n", "import net.imglib2.interpolation.randomaccess.FloorInterpolatorFactory\n", "import net.imglib2.RandomAccessibleInterval\n", "\n", "/* Utility Functions */\n", "tile = { images ->\n", " int[] gridLayout = images[0] in List ?\n", " [images[0].size, images.size] : // 2D images list\n", " [images.size] // 1D images list\n", " RandomAccessibleInterval[] rais = images.flatten()\n", " ij.notebook().mosaic(gridLayout, rais)\n", "}\n", "\n", "table_image = { array ->\n", " img = ij.op().create().kernel(array as double[][], new FloatType())\n", " ij.op().run(\"transform.scaleView\", img,\n", " [32,32] as double[],\n", " new FloorInterpolatorFactory()\n", " ) \n", "}\n", "\n", "zoomedView = { img, factor ->\n", " ij.op().run(\"transform.scaleView\", img,\n", " [factor,factor] as double[],\n", " new FloorInterpolatorFactory()\n", " ) \n", "}\n", "null" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Filtering\n", "\n", "* Convolution and kernels\n", "* Linear filters\n", " * Mean filter\n", " * Gauss filter\n", " * Edge-enhancing filters\n", "* Non-linear filters\n", " * Median filter\n", " * Minimum and maximum filters\n", "* Binary morphological operations\n", "* Skeletonization\n", "* Filtering in the frequency domain\n" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Impulse response" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "Let's look at a point light source (ok, for better visibility, it's a square):" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "hide_input": true, "scrolled": false, "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "text/html": [ "
Original
" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import net.imglib2.type.numeric.real.FloatType\n", "pixels = [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n", "image = ij.op().run(\"create.kernel\", pixels, new FloatType())\n", "ij.notebook().display([[\"Original\": zoomedView(image,16)]])" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "-" } }, "source": [ "What happens if we look at this point through a microscope?" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "The microscope optics lead to a blurred image on our camera chip:" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "hide_input": true, "scrolled": true, "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "text/html": [ "
OriginalImage
" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gaussKernel = ij.op().run(\"create.kernelGauss\", [1,1])\n", "// zoomedView(gaussKernel, 16)\n", "result = ij.op().run(\"filter.convolve\", image, gaussKernel)\n", "ij.notebook().display([[\"Original\": zoomedView(image,16), \"Image\": zoomedView(result,16)]])" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "-" } }, "source": [ " " ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "What happens if we have two points close together?" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "hide_input": true, "scrolled": false, "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "text/html": [ "
One pointImage of one pointTwo points
" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import net.imglib2.type.numeric.real.FloatType\n", "pixels = [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n", "image2 = ij.op().run(\"create.kernel\", pixels, new FloatType())\n", "ij.notebook().display([[\"One point\": zoomedView(image,16), \"Image of one point\": zoomedView(result,16), \"Two points\": zoomedView(image2,16)]])" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "-" } }, "source": [ " " ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "The microscope optics lead to a blurred image on our camera chip:" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "hide_input": true, "scrolled": true, "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "text/html": [ "
One pointImage of one pointTwo pointsImage of two points
" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gaussKernel = ij.op().run(\"create.kernelGauss\", [1,1])\n", "// zoomedView(gaussKernel, 16)\n", "result2 = ij.op().run(\"filter.convolve\", image2, gaussKernel)\n", "ij.notebook().display([[\"One point\": zoomedView(image,16), \"Image of one point\": zoomedView(result,16), \"Two points\": zoomedView(image2,16), \"Image of two points\": zoomedView(result2,16)]])" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "-" } }, "source": [ "..." ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Convolution\n", "\n", "In digital images, we can model this process by defining a **pixel neighborhood**\n", "\n", "$\\begin{bmatrix}\n", "i_{-1,-1} & i_{0,-1} & i_{1,-1}\\\\\n", "i_{-1,0} & i_{0,0} & i_{1,0}\\\\\n", "i_{-1,1} & i_{-1,1} & i_{1,1}\n", "\\end{bmatrix}$\n", "\n", "... and multiplying every pixel in the neighborhood with a certain **weight**, defined in a **kernel**:\n", "\n", "\n", "$\\begin{bmatrix}\n", "k_{1,1} & k_{2,1} & k_{3,1} \\\\\n", "k_{1,2} & k_{2,2} & k_{3,2} \\\\\n", "k_{1,3} & k_{2,3} & k_{3,3}\n", "\\end{bmatrix}$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Convolution\n", "\n", "For example, if we define a 3x3 neighborhood and a kernel full of `1`s: $\\begin{bmatrix}\n", "1 & 1 & 1 \\\\\n", "1 & 1 & 1 \\\\\n", "1 & 1 & 1\n", "\\end{bmatrix}$\n", "\n", "... each pixel in the result will be the sum of all the pixels in the neighborhood:\n", "\n", "$result_{0,0} = i_{-1,-1} + i_{0,-1} + i_{1,-1} + i_{-1,0} + i_{0,0} + i_{1,0} + i_{-1,1} + i_{-1,1} + i_{1,1}$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Convolution\n", "\n", "Digital image convolution is a process where each pixel is assigned the result of a matrix multiplication:\n", "\n", "$result = (i)mage * (k)ernel$\n", "\n", "$result = \\begin{bmatrix}\n", "i_{1,1} & i_{2,1} & i_{3,1} & i_{4,1} & i_{5,1}\\\\\n", "i_{1,2} & i_{2,2} & i_{3,2} & i_{4,2} & i_{5,2}\\\\\n", "i_{1,3} & i_{2,3} & \\color{red}{i_{3,3}} & i_{4,3} & i_{5,3}\\\\\n", "i_{1,3} & i_{2,4} & i_{3,4} & i_{4,4} & i_{5,4}\\\\\n", "i_{1,3} & i_{2,5} & i_{3,5} & i_{4,5} & i_{5,5}\n", "\\end{bmatrix} * \\begin{bmatrix}\n", "k_{1,1} & k_{2,1} & k_{3,1} \\\\\n", "k_{1,2} & k_{2,2} & k_{3,2} \\\\\n", "k_{1,3} & k_{2,3} & k_{3,3}\n", "\\end{bmatrix}\n", "$\n", "\n", "$result_{3,3} = i_{2,2} k_{3,3} + i_{3,2} k_{2,3} + i_{4,2} k_{1,3} + i_{2,3} k_{3,2} + ...$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Linear filters\n", "\n", "* In ImageJ, convolution with arbitrary kernels can be done via *Process > Filters > Convolve...*" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Identity: $\\begin{bmatrix}\n", "0 & 0 & 0 \\\\\n", "0 & 1 & 0 \\\\\n", "0 & 0 & 0\n", "\\end{bmatrix}$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Mean: $\\begin{bmatrix}\n", "\\frac{1}{9} & \\frac{1}{9} & \\frac{1}{9} \\\\\n", "\\frac{1}{9} & \\frac{1}{9} & \\frac{1}{9} \\\\\n", "\\frac{1}{9} & \\frac{1}{9} & \\frac{1}{9}\n", "\\end{bmatrix}$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Sobel: $\\begin{bmatrix}\n", "-1 & 0 & 1 \\\\\n", "-2 & 0 & 2 \\\\\n", "-1 & 0 & 1\n", "\\end{bmatrix}$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Laplace: $\\begin{bmatrix}\n", "0 & 1 & 0 \\\\\n", "1 & -4 & 1 \\\\\n", "0 & 1 & 0\n", "\\end{bmatrix}$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "#### Linear filters - Exercise\n", "\n", "* Open the *boats* sample image\n", "* How many implementations of Mean Filter are there in ImageJ? What's their difference?\n", "* Create a Mean filter with radius 10; what are the kernel dimensions?\n" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### The Gaussian Filter Kernel\n", "\n", "Approximate representation of a Gauss filter kernel\n", "\n", "$\\frac{1}{256}\n", "\\begin{bmatrix}\n", "1 & 4 & 6 & 4 & 1 \\\\\n", "4 & 16 & 24 & 16 & 4 \\\\\n", "6 & 24 & 36 & 24 & 6 \\\\\n", "4 & 16 & 24 & 16 & 4 \\\\\n", "1 & 4 & 6 & 4 & 1\n", "\\end{bmatrix}$" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "#### Linear filters in the ImageJ menu\n", "\n", "* [*Process*](https://imagej.net/docs/guide/146-29.html#toc-Section-29)\n", " * *Smooth*\n", " * *Sharpen*\n", " * *Find Edges*\n", " * [*Filters*](https://imagej.net/docs/guide/146-29.html#toc-Subsection-29.11)\n", " * *Gaussian Blur...*\n", " * *Mean...*\n", " " ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Rank filters (non-linear filters)\n", "\n", "* Median filter\n", "* Minimum filter\n", "* Maximum filter\n" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Binary morphological operations\n", "\n", "* Dilate\n", "* Erode\n", "* Open (Erode, then Dilate)\n", "* Close (Dilate, then Erode)\n", "\n", "https://imagej.net/docs/guide/146-29.html#toc-Subsection-29.8\n", "\n", "https://en.wikipedia.org/wiki/Mathematical_morphology\n" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "#### Morphological operations - Exercise\n", "\n", "* Open the FMI logo (*FMI > Teaching > FMI Logo*)\n", "* Resize the canvas to 100 x 50\n", "* Use *MorphoLibJ > Morphological Filters* to test various structuring elements" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Skeletonization\n", "\n", "https://imagej.net/docs/guide/146-29.html#sub:Skeletonize\n" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "### Filtering in the Frequency Domain\n", "\n", "https://imagej.net/docs/guide/146-29.html#toc-Subsection-29.10\n" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "Questions ?\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "image = ij.io().open(\"https://imagej.net/images/blobs.gif\")\n", "cropped = ij.op().run(\"hyperSliceView\", image, 2, 0)\n", "dims = new long[2]\n", "cropped.dimensions(dims)\n", "dims" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "import net.imglib2.type.numeric.real.FloatType\n", "kernelArray = [[0, 0, 1, 0, 0],\n", " [0, 0, 1, 0, 0],\n", " [1, 1, -8, 1, 1],\n", " [0, 0, 1, 0, 0],\n", " [0, 0, 1, 0, 0]]\n", "\n", "kernelArray2 = [[-1, 0, 1],\n", " [-2, 0, 2],\n", " [-1, 0, 1]]\n", "\n", "kernel = ij.op().run(\"create.kernel\", kernelArray2, new FloatType())\n", "\n", "zoomedView(kernel, 32)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "//extended = ij.op().run(\"intervalView\", ij.op().run(\"extendBorderView\", cropped), cropped)\n", "result = ij.op().run(\"filter.convolve\", cropped, kernel)\n", "ij.notebook().display([[\"Input\": cropped, \"Output\": result]])" ] } ], "metadata": { "celltoolbar": "Slideshow", "kernelspec": { "display_name": "Groovy", "language": "groovy", "name": "groovy" }, "language_info": { "codemirror_mode": "groovy", "file_extension": ".groovy", "mimetype": "", "name": "Groovy", "nbconverter_exporter": "", "version": "2.4.3" }, "rise": { "auto_select": "none", "autolaunch": true, "scroll": true, "transition": "none" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": false, "sideBar": false, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": { "height": "371.717px", "left": "10px", "top": "150px", "width": "338px" }, "toc_section_display": false, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }