<!DOCTYPE html>
<html>
<head>
  <meta name="databricks-html-version" content="1">
<title>036_IntroductionToMagellan - Databricks</title>

<meta charset="utf-8">
<meta name="google" content="notranslate">
<meta http-equiv="Content-Language" content="en">
<meta http-equiv="Content-Type" content="text/html; charset=UTF8">
<link rel="stylesheet"
  href="https://fonts.googleapis.com/css?family=Source+Code+Pro:400,700">

<link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/css/bootstrap.min.css">
<link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/jquery-ui-bundle/jquery-ui.min.css">
<link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/main.css">
<link rel="stylesheet" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/print.css" media="print">
<link rel="icon" type="image/png" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/img/favicon.ico"/>
<script>window.settings = {"sparkDocsSearchGoogleCx":"004588677886978090460:_rj0wilqwdm","dbcForumURL":"http://forums.databricks.com/","dbfsS3Host":"https://databricks-prod-storage-sydney.s3.amazonaws.com","enableThirdPartyApplicationsUI":false,"enableClusterAcls":false,"notebookRevisionVisibilityHorizon":0,"enableTableHandler":true,"isAdmin":true,"enableLargeResultDownload":false,"nameAndEmail":"Raazesh Sainudiin (r.sainudiin@math.canterbury.ac.nz)","enablePresentationTimerConfig":true,"enableFullTextSearch":true,"enableElasticSparkUI":true,"clusters":true,"hideOffHeapCache":false,"applications":false,"useStaticGuide":false,"fileStoreBase":"FileStore","configurableSparkOptionsSpec":[{"keyPattern":"spark\\.kryo(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.kryo.*","valuePatternDisplay":"*","description":"Configuration options for Kryo serialization"},{"keyPattern":"spark\\.io\\.compression\\.codec","valuePattern":"(lzf|snappy|org\\.apache\\.spark\\.io\\.LZFCompressionCodec|org\\.apache\\.spark\\.io\\.SnappyCompressionCodec)","keyPatternDisplay":"spark.io.compression.codec","valuePatternDisplay":"snappy|lzf","description":"The codec used to compress internal data such as RDD partitions, broadcast variables and shuffle outputs."},{"keyPattern":"spark\\.serializer","valuePattern":"(org\\.apache\\.spark\\.serializer\\.JavaSerializer|org\\.apache\\.spark\\.serializer\\.KryoSerializer)","keyPatternDisplay":"spark.serializer","valuePatternDisplay":"org.apache.spark.serializer.JavaSerializer|org.apache.spark.serializer.KryoSerializer","description":"Class to use for serializing objects that will be sent over the network or need to be cached in serialized form."},{"keyPattern":"spark\\.rdd\\.compress","valuePattern":"(true|false)","keyPatternDisplay":"spark.rdd.compress","valuePatternDisplay":"true|false","description":"Whether to compress serialized RDD partitions (e.g. for StorageLevel.MEMORY_ONLY_SER). Can save substantial space at the cost of some extra CPU time."},{"keyPattern":"spark\\.speculation","valuePattern":"(true|false)","keyPatternDisplay":"spark.speculation","valuePatternDisplay":"true|false","description":"Whether to use speculation (recommended off for streaming)"},{"keyPattern":"spark\\.es(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"es(\\.([^\\.]+))+","valuePattern":".*","keyPatternDisplay":"es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"spark\\.(storage|shuffle)\\.memoryFraction","valuePattern":"0?\\.0*([1-9])([0-9])*","keyPatternDisplay":"spark.(storage|shuffle).memoryFraction","valuePatternDisplay":"(0.0,1.0)","description":"Fraction of Java heap to use for Spark's shuffle or storage"},{"keyPattern":"spark\\.streaming\\.backpressure\\.enabled","valuePattern":"(true|false)","keyPatternDisplay":"spark.streaming.backpressure.enabled","valuePatternDisplay":"true|false","description":"Enables or disables Spark Streaming's internal backpressure mechanism (since 1.5). This enables the Spark Streaming to control the receiving rate based on the current batch scheduling delays and processing times so that the system receives only as fast as the system can process. Internally, this dynamically sets the maximum receiving rate of receivers. This rate is upper bounded by the values `spark.streaming.receiver.maxRate` and `spark.streaming.kafka.maxRatePerPartition` if they are set."},{"keyPattern":"spark\\.streaming\\.receiver\\.maxRate","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.receiver.maxRate","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which each receiver will receive data. Effectively, each stream will consume at most this number of records per second. Setting this configuration to 0 or a negative number will put no limit on the rate. See the deployment guide in the Spark Streaming programing guide for mode details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRatePerPartition","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRatePerPartition","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which data will be read from each Kafka partition when using the Kafka direct stream API introduced in Spark 1.3. See the Kafka Integration guide for more details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRetries","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRetries","valuePatternDisplay":"numeric","description":"Maximum number of consecutive retries the driver will make in order to find the latest offsets on the leader of each partition (a default value of 1 means that the driver will make a maximum of 2 attempts). Only applies to the Kafka direct stream API introduced in Spark 1.3."},{"keyPattern":"spark\\.streaming\\.ui\\.retainedBatches","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.ui.retainedBatches","valuePatternDisplay":"numeric","description":"How many batches the Spark Streaming UI and status APIs remember before garbage collecting."}],"enableReactNotebookComments":true,"enableResetPassword":true,"enableJobsSparkUpgrade":true,"sparkVersions":[{"key":"1.3.x-ubuntu15.10","displayName":"Spark 1.3.0","packageLabel":"spark-1.3-jenkins-ip-10-30-9-162-U0c2673ac85-Sa2ee4664b2-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.4.x-ubuntu15.10","displayName":"Spark 1.4.1","packageLabel":"spark-1.4-jenkins-ip-10-30-9-162-U0c2673ac85-S33a1e4b9c6-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.6.x-ubuntu15.10","displayName":"Spark 1.6.0","packageLabel":"spark-1.6-jenkins-ip-10-30-9-162-U0c2673ac85-Scabba801f3-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"master","displayName":"Spark master (dev)","packageLabel":"","upgradable":true,"deprecated":false,"customerVisible":false}],"enableRestrictedClusterCreation":false,"enableFeedback":false,"defaultNumWorkers":8,"serverContinuationTimeoutMillis":10000,"driverStderrFilePrefix":"stderr","driverStdoutFilePrefix":"stdout","enableSparkDocsSearch":true,"prefetchSidebarNodes":true,"sparkHistoryServerEnabled":true,"sanitizeMarkdownHtml":true,"enableIPythonImportExport":true,"enableNotebookHistoryDiffing":true,"branch":"2.12.3","accountsLimit":-1,"enableNotebookGitBranching":true,"local":false,"displayDefaultContainerMemoryGB":6,"deploymentMode":"production","useSpotForWorkers":false,"enableUserInviteWorkflow":false,"enableStaticNotebooks":true,"dbcGuideURL":"#workspace/databricks_guide/00 Welcome to Databricks","enableCssTransitions":true,"pricingURL":"https://databricks.com/product/pricing","enableClusterAclsConfig":false,"orgId":0,"enableNotebookGitVersioning":true,"files":"files/","enableDriverLogsUI":true,"disableLegacyDashboards":false,"enableWorkspaceAclsConfig":true,"dropzoneMaxFileSize":4096,"enableNewDashboardViews":false,"driverLog4jFilePrefix":"log4j","enableMavenLibraries":true,"displayRowLimit":1000,"defaultSparkVersion":{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},"clusterPublisherRootId":5,"enableLatestJobRunResultPermalink":true,"disallowAddingAdmins":false,"enableSparkConfUI":true,"enableOrgSwitcherUI":false,"clustersLimit":-1,"enableJdbcImport":true,"logfiles":"logfiles/","enableWebappSharding":false,"enableClusterDeltaUpdates":true,"csrfToken":"4c37e4ff-a908-4b05-8c61-2b76819fa34c","useFixedStaticNotebookVersionForDevelopment":false,"enableBasicReactDialogBoxes":true,"requireEmailUserName":true,"enableDashboardViews":false,"dbcFeedbackURL":"http://feedback.databricks.com/forums/263785-product-feedback","enableWorkspaceAclService":true,"someName":"Raazesh Sainudiin","enableWorkspaceAcls":true,"gitHash":"0c2673ac858e227cad536fdb45d140aeded238db","userFullname":"Raazesh Sainudiin","enableClusterCreatePage":false,"enableImportFromUrl":true,"enableMiniClusters":false,"enableWebSocketDeltaUpdates":true,"enableDebugUI":false,"showHiddenSparkVersions":false,"allowNonAdminUsers":true,"userId":100005,"dbcSupportURL":"","staticNotebookResourceUrl":"https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/","enableSparkPackages":true,"enableHybridClusterType":false,"enableNotebookHistoryUI":true,"availableWorkspaces":[{"name":"Workspace 0","orgId":0}],"enableFolderHtmlExport":true,"enableSparkVersionsUI":true,"databricksGuideStaticUrl":"","enableHybridClusters":true,"notebookLoadingBackground":"#fff","enableNewJobRunDetailsPage":true,"enableDashboardExport":true,"user":"r.sainudiin@math.canterbury.ac.nz","enableServerAutoComplete":true,"enableStaticHtmlImport":true,"defaultMemoryPerContainerMB":6000,"enablePresenceUI":true,"tablesPublisherRootId":7,"enableNewInputWidgetUI":false,"accounts":true,"enableNewProgressReportUI":true,"defaultCoresPerContainer":4};</script>
<script>var __DATABRICKS_NOTEBOOK_MODEL = {"version":"NotebookV1","origId":116350,"name":"036_IntroductionToMagellan","language":"scala","commands":[{"version":"CommandV1","origId":116352,"guid":"ee48485c-f52e-44c2-80b6-f17898d368cc","subtype":"command","commandType":"auto","position":0.5,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n### prepared by Dillon George, [Raazesh Sainudiin](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [Sivanand Sivaram](https://www.linkedin.com/in/sivanand)\n\n*supported by* [![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/databricks_logoTM_200px.png)](https://databricks.com/)\nand \n[![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/AWS_logoTM_200px.png)](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"5edb52cd-a2c1-4ca5-98c2-a1e18eea22ae"},{"version":"CommandV1","origId":129744,"guid":"04873213-c4e7-46ab-8a6a-c6641b8783f7","subtype":"command","commandType":"auto","position":0.75,"command":"%md\nThe [html source url](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/db/week10/036_IntroductionToMagellan.html) of this databricks notebook and its recorded Uji ![Image of Uji, Dogen's Time-Being](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/UjiTimeBeingDogen.png \"uji\"):\n\n[![sds/uji/week2/week10/036_IntroductionToMagellan](http://img.youtube.com/vi/0wKxVfeBQBc/0.jpg)](https://www.youtube.com/v/0wKxVfeBQBc?rel=0&autoplay=1&modestbranding=1&start=754&end=2056)\n","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"da4c5b38-74f5-4bd0-ad9b-ebdc8ab32ea9"},{"version":"CommandV1","origId":116353,"guid":"b7953a7d-95c4-44f1-8d4c-c3dff0b1b887","subtype":"command","commandType":"auto","position":1.0,"command":"%md\n# Introduction to Magellan for Scalable Geospatial Analytics\n\nThis is a minor  augmentation of Ram Harsha's Magellan code blogged here:\n* [magellan geospatial analytics in spark](http://hortonworks.com/blog/magellan-geospatial-analytics-in-spark/)\n\nFirst you need to attach the following two libraries:\n* the magellan library \n  * Here we are using Spark 1.5.2 and the magellan library as a jar file that was built from [github magellan source](https://github.com/harsha2010/magellan).\n    * we could not use the spark package: [http://spark-packages.org/package/harsha2010/magellan](http://spark-packages.org/package/harsha2010/magellan).\n    * apparently magellan will be spark 2.0 ready in the future).\n* and also the esri-geometry-api-1.2.1 library to databricks cluster.\n\nSee [using 3rd party libraries in databricks](https://databricks.com/blog/2015/07/28/using-3rd-party-libraries-in-databricks-spark-packages-and-maven-libraries.html) for help on attaching libraries in databricks.\n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451992569E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"22103f42-9502-4c55-b252-207d8f1bb996"},{"version":"CommandV1","origId":116354,"guid":"7f82c37d-7283-4f7b-b81b-73d442b7f19b","subtype":"command","commandType":"auto","position":1.5,"command":"%md\n## Magellan-Spark as a Scalable Geospatial Analytics Engine \n\nHOMEWORK: Watch the [magellan presentation by Ram Harsha (Hortonworks) in Spark Summit East 2016](https://spark-summit.org/east-2016/events/magellan-spark-as-a-geospatial-analytics-engine/).\n\n[![Ram harsha's Magellan Spark Summit East 2016 Talk]](http://img.youtube.com/vi/1lF1oSjxMT4/0.jpg)](https://www.youtube.com/watch?v=1lF1oSjxMT4)\n\n  \nOther resources for magellan:\n* [Ram's blog in HortonWorks](http://hortonworks.com/blog/magellan-geospatial-analytics-in-spark/) and the [ZeppelinHub view of the demo code in video above](https://www.zeppelinhub.com/viewer/notebooks/aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL2hvcnRvbndvcmtzLWdhbGxlcnkvemVwcGVsaW4tbm90ZWJvb2tzL21hc3Rlci8yQjRUV0dDOE0vbm90ZS5qc29u)\n* [Magellan as Spark project](http://spark-packages.org/package/harsha2010/magellan) and [Magellan github source](https://github.com/harsha2010/magellan)\n* [shape files](https://en.wikipedia.org/wiki/Shapefile) developed by Environmental Systems Research Institute [(ESRI)](https://en.wikipedia.org/wiki/Esri). See ESRI's [what is a geospatial shape file?](https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf)\n* magellan builds on [http://esri.github.io/](http://esri.github.io/) a leading opensource geospatial library\n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451992616E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9053de6c-9939-4215-8360-1644b819c946"},{"version":"CommandV1","origId":116355,"guid":"fa801670-5325-4e39-b935-6b0488965320","subtype":"command","commandType":"auto","position":1.625,"command":"%md\n\n## Do we need one more geospatial analytics library?\n\n**(watch later 4 minutes and 10 seconds)**:\n\n[![Spark Summit East 2016 - What is Geospatial Analytics by Ram Sri Harsha](http://img.youtube.com/vi/1lF1oSjxMT4/0.jpg)](https://www.youtube.com/v/1lF1oSjxMT4?rel=0&autoplay=1&modestbranding=1&start=318&end=568)\n\nFrom [Ram's slide 4 of this Spark Summit East 2016 talk at slideshare](http://www.slideshare.net/SparkSummit/magellanspark-as-a-geospatial-analytics-engine-by-ram-sriharsha):\n\n* Spatial Analytics at scale is challenging \n  * Simplicity + Scalability = Hard \n* Ancient Data Formats \n  * metadata, indexing not handled well, inefficient storage \n* Geospatial Analytics is not simply Business Intelligence anymore \n  * Statistical + Machine Learning being leveraged in geospatial \n* Now is the time to do it! \n  * Explosion of mobile data \n  * Finer granularity of data collection for geometries \n  * Analytics stretching the limits of traditional approaches \n  * Spark SQL + Catalyst + Tungsten makes extensible SQL engines easier than ever before! \n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"edfb245b-ca7c-455c-8f57-772971462f50"},{"version":"CommandV1","origId":118513,"guid":"95d67756-c52c-4ff7-9a89-651235da9bec","subtype":"command","commandType":"auto","position":1.6875,"command":"%md\n## Nuts and Bolts of Magellan","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7ab3469d-cc36-4eb7-996f-6706fe9cb584"},{"version":"CommandV1","origId":116356,"guid":"742f0978-6a1e-4a78-b134-e21956a51fe7","subtype":"command","commandType":"auto","position":1.75,"command":"%md \nLet us first import what we will need for our geo-spatial analysis below.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451992679E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"801bad9b-fcdf-4530-9887-66023491fdac"},{"version":"CommandV1","origId":116357,"guid":"684bce6f-6edb-4edb-a7c6-4be5eadcf82b","subtype":"command","commandType":"auto","position":2.0,"command":"// import statements are below NOTE: this magellan needs spark 1.5.1 and the jar was built from source and attached as a snapshot here\nimport magellan.{Point, Polygon, PolyLine}\nimport magellan.coord.NAD83\nimport org.apache.spark.sql.magellan.MagellanContext\nimport org.apache.spark.sql.magellan.dsl.expressions._\nimport org.apache.spark.sql.Row\nimport org.apache.spark.sql.types._\nimport org.apache.spark.sql.functions._ // this is needed for sql functions like explode, etc.\n\nimport java.sql.Timestamp\nimport java.text.SimpleDateFormat","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">import magellan.{Point, Polygon, PolyLine}\nimport magellan.coord.NAD83\nimport org.apache.spark.sql.magellan.MagellanContext\nimport org.apache.spark.sql.magellan.dsl.expressions._\nimport org.apache.spark.sql.Row\nimport org.apache.spark.sql.types._\nimport org.apache.spark.sql.functions._\nimport java.sql.Timestamp\nimport java.text.SimpleDateFormat\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463633697859E12,"submitTime":1.463633542656E12,"finishTime":1.463633699601E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"ecafe5da-5334-46ec-b251-89752eedb1bc"},{"version":"CommandV1","origId":116358,"guid":"9eac0c6a-6f7b-4a6e-90cc-f1cc2509793e","subtype":"command","commandType":"auto","position":3.0,"command":"%md \n### Data Structure: Point","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451992782E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f0f5d0a3-441e-4c6d-9eaa-10646f90ea42"},{"version":"CommandV1","origId":116359,"guid":"41a40577-1618-481f-a6bc-686e6743a0d7","subtype":"command","commandType":"auto","position":4.0,"command":"val points = sc.parallelize(Seq((-1.0, -1.0), (-1.0, 1.0), (1.0, -1.0)))\n               .toDF(\"x\", \"y\")\n               .select(point($\"x\", $\"y\").as(\"point\"))","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">points: org.apache.spark.sql.DataFrame = [point: poin]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\">&lt;console&gt;:28: error: not found: value point\n                      .select(point($&quot;x&quot;, $&quot;y&quot;).as(&quot;point&quot;))\n                              ^\n</div>","error":null,"startTime":1.463633733454E12,"submitTime":1.463633578232E12,"finishTime":1.463633734489E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cf4901db-8dbe-46dc-98c8-a3f05110009c"},{"version":"CommandV1","origId":116360,"guid":"571fd63b-7c21-4ee0-af7e-c7a1beb8c515","subtype":"command","commandType":"auto","position":4.5,"command":"points.show(false)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-----------------+\n|point            |\n+-----------------+\n|Point(-1.0, -1.0)|\n|Point(-1.0, 1.0) |\n|Point(1.0, -1.0) |\n+-----------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Cancelled","error":null,"startTime":1.463633755241E12,"submitTime":1.463633600028E12,"finishTime":1.463633757777E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"873e0ae8-6562-4d6c-9e38-ab58a839b803"},{"version":"CommandV1","origId":116361,"guid":"0d222bb7-ba6b-4d5d-9543-bd80c5176382","subtype":"command","commandType":"auto","position":5.0,"command":"%md \n### Data Structure: Polygon","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451992971E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"1d77a284-d13b-4251-8c91-d0cdde1e00e3"},{"version":"CommandV1","origId":116362,"guid":"c61cc2e9-c865-432d-a34c-6a27e8a8d86b","subtype":"command","commandType":"auto","position":6.0,"command":"case class PolygonRecord(polygon: Polygon) // let's create a case class for magellan polygon\n\nval ring = Array(new Point(1.0, 1.0), new Point(1.0, -1.0),\n                 new Point(-1.0, -1.0), new Point(-1.0, 1.0),\n                 new Point(1.0, 1.0))","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">defined class PolygonRecord\nring: Array[magellan.Point] = Array(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0))\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"scala.reflect.internal.Types$TypeError: bad symbolic reference. A signature in Shape.class refers to term geometry","error":"<div class=\"ansiout\">in value com.core which is not available.\nIt may be completely missing from the current classpath, or the version on\nthe classpath might be incompatible with the version used when compiling Shape.class.\n\tat scala.reflect.internal.pickling.UnPickler$Scan.toTypeError(UnPickler.scala:847)\n\tat scala.reflect.internal.pickling.UnPickler$Scan$LazyTypeRef.complete(UnPickler.scala:854)\n\tat scala.reflect.internal.pickling.UnPickler$Scan$LazyTypeRef.load(UnPickler.scala:863)\n\tat scala.reflect.internal.Symbols$Symbol.typeParams(Symbols.scala:1489)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$scala$tools$nsc$transform$SpecializeTypes$$normalizeMember$1.apply(SpecializeTypes.scala:798)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$scala$tools$nsc$transform$SpecializeTypes$$normalizeMember$1.apply(SpecializeTypes.scala:798)\n\tat scala.reflect.internal.SymbolTable.atPhase(SymbolTable.scala:207)\n\tat scala.reflect.internal.SymbolTable.beforePhase(SymbolTable.scala:215)\n\tat scala.tools.nsc.transform.SpecializeTypes.scala$tools$nsc$transform$SpecializeTypes$$normalizeMember(SpecializeTypes.scala:797)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$22.apply(SpecializeTypes.scala:751)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$22.apply(SpecializeTypes.scala:749)\n\tat scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)\n\tat scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251)\n\tat scala.collection.immutable.List.foreach(List.scala:318)\n\tat scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251)\n\tat scala.collection.AbstractTraversable.flatMap(Traversable.scala:105)\n\tat scala.tools.nsc.transform.SpecializeTypes.specializeClass(SpecializeTypes.scala:749)\n\tat scala.tools.nsc.transform.SpecializeTypes.transformInfo(SpecializeTypes.scala:1172)\n\tat scala.tools.nsc.transform.InfoTransform$Phase$$anon$1.transform(InfoTransform.scala:38)\n\tat scala.reflect.internal.Symbols$Symbol.rawInfo(Symbols.scala:1321)\n\tat scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1241)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$transformInfo$1$$anonfun$apply$36.apply(SpecializeTypes.scala:1164)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$transformInfo$1$$anonfun$apply$36.apply(SpecializeTypes.scala:1164)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.immutable.List.foreach(List.scala:318)\n\tat scala.collection.TraversableLike$class.map(TraversableLike.scala:244)\n\tat scala.collection.AbstractTraversable.map(Traversable.scala:105)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$transformInfo$1.apply(SpecializeTypes.scala:1164)\n\tat scala.tools.nsc.transform.SpecializeTypes$$anonfun$transformInfo$1.apply(SpecializeTypes.scala:1164)\n\tat scala.reflect.internal.SymbolTable.atPhase(SymbolTable.scala:207)\n\tat scala.reflect.internal.SymbolTable.afterPhase(SymbolTable.scala:216)\n\tat scala.tools.nsc.transform.SpecializeTypes.transformInfo(SpecializeTypes.scala:1164)\n\tat scala.tools.nsc.transform.InfoTransform$Phase$$anon$1.transform(InfoTransform.scala:38)\n\tat scala.reflect.internal.Symbols$Symbol.rawInfo(Symbols.scala:1321)\n\tat scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1241)\n\tat scala.reflect.internal.Symbols$Symbol.isDerivedValueClass(Symbols.scala:658)\n\tat scala.reflect.internal.transform.Erasure$ErasureMap.applyInArray(Erasure.scala:160)\n\tat scala.reflect.internal.transform.Erasure$ErasureMap$$anonfun$apply$3.apply(Erasure.scala:127)\n\tat scala.reflect.internal.transform.Erasure$ErasureMap$$anonfun$apply$3.apply(Erasure.scala:127)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.immutable.List.foreach(List.scala:318)\n\tat scala.collection.TraversableLike$class.map(TraversableLike.scala:244)\n\tat scala.collection.AbstractTraversable.map(Traversable.scala:105)\n\tat scala.reflect.internal.transform.Erasure$ErasureMap.apply(Erasure.scala:127)\n\tat scala.reflect.internal.transform.Erasure$ErasureMap.apply(Erasure.scala:144)\n\tat scala.reflect.internal.transform.Erasure$class.specialErasure(Erasure.scala:209)\n\tat scala.tools.nsc.transform.Erasure.specialErasure(Erasure.scala:14)\n\tat scala.reflect.internal.transform.Erasure$class.transformInfo(Erasure.scala:380)\n\tat scala.tools.nsc.transform.Erasure.transformInfo(Erasure.scala:329)\n\tat scala.tools.nsc.transform.InfoTransform$Phase$$anon$1.transform(InfoTransform.scala:38)\n\tat scala.reflect.internal.Symbols$Symbol.rawInfo(Symbols.scala:1321)\n\tat scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1241)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anonfun$scala$tools$nsc$transform$Erasure$ErasureTransformer$$checkNoDoubleDefs$1$$anonfun$apply$mcV$sp$2.apply(Erasure.scala:931)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anonfun$scala$tools$nsc$transform$Erasure$ErasureTransformer$$checkNoDoubleDefs$1$$anonfun$apply$mcV$sp$2.apply(Erasure.scala:931)\n\tat scala.reflect.internal.Scopes$Scope.foreach(Scopes.scala:315)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anonfun$scala$tools$nsc$transform$Erasure$ErasureTransformer$$checkNoDoubleDefs$1.apply(Erasure.scala:931)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anonfun$scala$tools$nsc$transform$Erasure$ErasureTransformer$$checkNoDoubleDefs$1.apply(Erasure.scala:931)\n\tat scala.reflect.internal.SymbolTable.atPhase(SymbolTable.scala:207)\n\tat scala.reflect.internal.SymbolTable.afterPhase(SymbolTable.scala:216)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer.scala$tools$nsc$transform$Erasure$ErasureTransformer$$checkNoDoubleDefs(Erasure.scala:931)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.preErase(Erasure.scala:1245)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1280)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1276)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$1.apply(TypingTransformers.scala:44)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer.transformTemplate(Trees.scala:2904)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1280)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$4.apply(Trees.scala:1279)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1278)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:48)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1030)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2927)\n\tat scala.reflect.api.Trees$Transformer$$anonfun$transformStats$1.apply(Trees.scala:2925)\n\tat scala.collection.immutable.List.loop$1(List.scala:170)\n\tat scala.collection.immutable.List.mapConserve(List.scala:186)\n\tat scala.reflect.api.Trees$Transformer.transformStats(Trees.scala:2925)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)\n\tat scala.reflect.internal.Trees$$anonfun$itransform$7.apply(Trees.scala:1298)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:28)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:19)\n\tat scala.reflect.internal.Trees$class.itransform(Trees.scala:1297)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.internal.SymbolTable.itransform(SymbolTable.scala:13)\n\tat scala.reflect.api.Trees$Transformer.transform(Trees.scala:2897)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.scala$tools$nsc$transform$TypingTransformers$TypingTransformer$$super$transform(TypingTransformers.scala:44)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$2.apply(TypingTransformers.scala:46)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer$$anonfun$transform$2.apply(TypingTransformers.scala:46)\n\tat scala.reflect.api.Trees$Transformer.atOwner(Trees.scala:2936)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.atOwner(TypingTransformers.scala:34)\n\tat scala.tools.nsc.transform.TypingTransformers$TypingTransformer.transform(TypingTransformers.scala:46)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer$$anon$1.transform(Erasure.scala:1292)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer.transform(Erasure.scala:1302)\n\tat scala.tools.nsc.transform.Erasure$ErasureTransformer.transform(Erasure.scala:888)\n\tat scala.tools.nsc.ast.Trees$Transformer.transformUnit(Trees.scala:227)\n\tat scala.tools.nsc.transform.Transform$Phase.apply(Transform.scala:30)\n\tat scala.tools.nsc.Global$GlobalPhase.applyPhase(Global.scala:464)\n\tat scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)\n\tat scala.tools.nsc.Global$GlobalPhase$$anonfun$run$1.apply(Global.scala:431)\n\tat scala.collection.Iterator$class.foreach(Iterator.scala:727)\n\tat scala.collection.AbstractIterator.foreach(Iterator.scala:1157)\n\tat scala.tools.nsc.Global$GlobalPhase.run(Global.scala:431)\n\tat scala.tools.nsc.Global$Run.compileUnitsInternal(Global.scala:1583)\n\tat scala.tools.nsc.Global$Run.compileUnits(Global.scala:1557)\n\tat scala.tools.nsc.Global$Run.compileSources(Global.scala:1553)\n\tat org.apache.spark.repl.SparkIMain.org$apache$spark$repl$SparkIMain$$compileSourcesKeepingRun(SparkIMain.scala:663)\n\tat org.apache.spark.repl.SparkIMain$ReadEvalPrint.compileAndSaveRun(SparkIMain.scala:1138)\n\tat org.apache.spark.repl.SparkIMain$ReadEvalPrint.compile(SparkIMain.scala:1094)\n\tat org.apache.spark.repl.SparkIMain$Request.compile$lzycompute(SparkIMain.scala:1294)\n\tat org.apache.spark.repl.SparkIMain$Request.compile(SparkIMain.scala:1289)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:870)\n\tat org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)\n\tat com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$repl$1.apply(ScalaDriverLocal.scala:130)\n\tat com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$repl$1.apply(ScalaDriverLocal.scala:130)\n\tat com.databricks.backend.daemon.driver.DriverLocal$TrapExitInternal$.trapExit(DriverLocal.scala:402)\n\tat com.databricks.backend.daemon.driver.DriverLocal$TrapExit$.apply(DriverLocal.scala:363)\n\tat com.databricks.backend.daemon.driver.ScalaDriverLocal.repl(ScalaDriverLocal.scala:130)\n\tat com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:177)\n\tat com.databricks.backend.daemon.driver.DriverWrapper$$anonfun$3.apply(DriverWrapper.scala:485)\n\tat com.databricks.backend.daemon.driver.DriverWrapper$$anonfun$3.apply(DriverWrapper.scala:485)\n\tat scala.util.Try$.apply(Try.scala:161)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:482)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:384)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:195)\n\tat java.lang.Thread.run(Thread.java:745)</div>","startTime":1.463633821149E12,"submitTime":1.463633665906E12,"finishTime":1.463633821912E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7a950d55-756b-4367-865c-369abfed6769"},{"version":"CommandV1","origId":116652,"guid":"ed8075be-e934-4b67-a950-2ccedcf55a5b","subtype":"command","commandType":"auto","position":6.5,"command":"%md \nThe `val` `ring` above encode the square polygon given by \\\\([-1,1]^2\\\\).","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b408b227-1d1c-4842-a823-32d311641bc5"},{"version":"CommandV1","origId":116363,"guid":"5e5a5bd5-6183-4b8a-a5cf-5f1523ac96d6","subtype":"command","commandType":"auto","position":7.0,"command":"// let's creat a data frame of polygons\nval polygons = sc.parallelize(Seq(\n    PolygonRecord(new Polygon(Array(0), ring))\n  )).toDF()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">polygons: org.apache.spark.sql.DataFrame = [polygon: polygo]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\">&lt;console&gt;:42: error: not found: value PolygonRecord\n           PolygonRecord(new Polygon(Array(0), ring))\n           ^\n</div>","error":null,"startTime":1.463633862782E12,"submitTime":1.463633707545E12,"finishTime":1.463633863153E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"50c7b5be-426c-4da2-a8f1-0d0a0c8ac734"},{"version":"CommandV1","origId":116364,"guid":"a996dfeb-5277-42f7-8e67-8ff277c1f85e","subtype":"command","commandType":"auto","position":8.0,"command":"polygons.show(false)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-------------------------------------------------------------------------------------------------------------------------------+\n|polygon                                                                                                                        |\n+-------------------------------------------------------------------------------------------------------------------------------+\n|Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n+-------------------------------------------------------------------------------------------------------------------------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\">&lt;console&gt;:41: error: not found: value polygons\n              polygons.show()\n              ^\n</div>","error":null,"startTime":1.463633866197E12,"submitTime":1.463633710789E12,"finishTime":1.46363386666E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"19e7bff0-c6bf-4fa6-9374-324959e8770c"},{"version":"CommandV1","origId":116365,"guid":"25be2bba-7b1d-44d3-b894-5d6cef9fdf97","subtype":"command","commandType":"auto","position":9.0,"command":"%md \n### Predicate: within","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451993153E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2c18dfa9-dffc-48e8-967d-531e8ca60276"},{"version":"CommandV1","origId":116366,"guid":"93f7201c-ebf8-4fdf-842a-ef24c4cf4e29","subtype":"command","commandType":"auto","position":10.0,"command":"polygons.select(point(0.5, 0.5) within $\"polygon\").show(false) //.count() ","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-------------------------------+\n|Within(shapeliteral(), polygon)|\n+-------------------------------+\n|true                           |\n+-------------------------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463633914221E12,"submitTime":1.463633758946E12,"finishTime":1.463633914558E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b93d959f-66ef-4733-97da-b3f721cb5965"},{"version":"CommandV1","origId":116367,"guid":"78222fc9-b555-493a-95ec-7430fe1ca882","subtype":"command","commandType":"auto","position":11.0,"command":"%md \n### Predicate: intersects","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451993294E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"77b9a8c3-77e1-49c2-b8cd-939fbe3f856b"},{"version":"CommandV1","origId":116651,"guid":"6f796622-375f-4165-8dba-33c9424b6b40","subtype":"command","commandType":"auto","position":11.25,"command":"%md\nThe `join` leverages SparkSQL's Catalyst and Tungsten.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"8ba43a02-89c8-4494-8bc8-461b0b19ab64"},{"version":"CommandV1","origId":116646,"guid":"ef505666-74de-40fc-bb98-be4d2c970a68","subtype":"command","commandType":"auto","position":11.5,"command":"points.join(polygons).show(false)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|point            |polygon                                                                                                                        |\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|Point(-1.0, -1.0)|Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(-1.0, 1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(1.0, -1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463633941122E12,"submitTime":1.463633785903E12,"finishTime":1.463633941673E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"30fb7ef6-0ba4-4577-ac61-11353408f711"},{"version":"CommandV1","origId":116368,"guid":"3032ae9f-06cc-4e0d-ac67-d01c2fa7ba18","subtype":"command","commandType":"auto","position":12.0,"command":"points.join(polygons).where($\"point\" intersects $\"polygon\").show(false) // all these points intersect the polygon","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|point            |polygon                                                                                                                        |\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|Point(-1.0, -1.0)|Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(-1.0, 1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(1.0, -1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\">&lt;console&gt;:51: error: value display is not a member of org.apache.spark.sql.DataFrame\n              points.join(polygons).where($&quot;point&quot; intersects $&quot;polygon&quot;).display()\n                                                                          ^\n</div>","error":null,"startTime":1.463633973601E12,"submitTime":1.463633818355E12,"finishTime":1.463633974206E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"bd2730d1-e3f0-487b-b20a-249035209ce6"},{"version":"CommandV1","origId":116650,"guid":"49f4108c-0692-4c11-a804-d7ba93462416","subtype":"command","commandType":"auto","position":12.25,"command":"%md\nLet's add another point \\\\( (-2,-2)\\\\) that lies outside our \\\\([-1,1]^2\\\\) polygon.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b97ea314-af28-4c7d-9369-fddb88f0740d"},{"version":"CommandV1","origId":116647,"guid":"e74cbb1b-bdcc-4503-bdd7-796b69112d31","subtype":"command","commandType":"auto","position":12.5,"command":"val morePoints = sc.parallelize(Seq((-1.0, -1.0), (-1.0, 1.0), (1.0, -1.0), (-2.0,-2.0)))\n               .toDF(\"x\", \"y\")\n               .select(point($\"x\", $\"y\").as(\"point\"))","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">morePoints: org.apache.spark.sql.DataFrame = [point: poin]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463633988354E12,"submitTime":1.463633833133E12,"finishTime":1.463633988589E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cf44ef28-6b58-4a88-b73d-90180bd381de"},{"version":"CommandV1","origId":116649,"guid":"90d3c764-f9d5-4401-b50a-da7a415b05ea","subtype":"command","commandType":"auto","position":12.625,"command":"morePoints.join(polygons).show(false)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|point            |polygon                                                                                                                        |\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|Point(-1.0, -1.0)|Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(-1.0, 1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(1.0, -1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(-2.0, -2.0)|Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463633997524E12,"submitTime":1.463633842296E12,"finishTime":1.463633997864E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"55eea1cc-afa4-47c9-9618-b354e0c39e5d"},{"version":"CommandV1","origId":116648,"guid":"a6418555-ecee-4229-b0da-e301c9e5772e","subtype":"command","commandType":"auto","position":12.75,"command":"morePoints.join(polygons).where($\"point\" intersects $\"polygon\").show(false) // (-2,-2) is not in the polygon","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|point            |polygon                                                                                                                        |\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n|Point(-1.0, -1.0)|Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(-1.0, 1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n|Point(1.0, -1.0) |Polygon(WrappedArray(0), WrappedArray(Point(1.0, 1.0), Point(1.0, -1.0), Point(-1.0, -1.0), Point(-1.0, 1.0), Point(1.0, 1.0)))|\n+-----------------+-------------------------------------------------------------------------------------------------------------------------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463634007552E12,"submitTime":1.463633852335E12,"finishTime":1.463634008278E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"6a733db9-224b-406c-8e51-09eba52543c9"},{"version":"CommandV1","origId":116369,"guid":"d22f37c8-3483-401a-9eca-cba0c872c365","subtype":"command","commandType":"auto","position":13.0,"command":"%md \n## Uber Dataset for the Demo done by Ram Harsha in Europe Spark Summit 2015\n\nFirst the datasets have to be loaded.  See the section below on **Downloading datasets and putting them in distributed file system** for doing this anew (This only needs to be done once if the data is persisted in the distributed file system).","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451993438E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"98fac39c-cf2f-45c6-a0f7-f9223fceb049"},{"version":"CommandV1","origId":116370,"guid":"593d2806-4ba7-4224-b10c-1f7efa3a71cc","subtype":"command","commandType":"auto","position":13.125,"command":"%md \nAfter downloading the data, we expect to have the following files in distributed file system (dbfs):\n\n* ```all.tsv``` is the file of all uber trajectories\n* ```SFNbhd``` is the directory containing SF neighborhood shape files.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451993551E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"28ee2bcd-f9bf-4f2a-86e0-89943e678953"},{"version":"CommandV1","origId":116371,"guid":"3a68eaef-e84f-4d3d-b854-c1e1d04fc05d","subtype":"command","commandType":"auto","position":13.25,"command":"display(dbutils.fs.ls(\"dbfs:/datasets/magellan/\")) // display the contents of the dbfs directory \"dbfs:/datasets/magellan/\"","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/datasets/magellan/SFNbhd/","SFNbhd/",0.0],["dbfs:/datasets/magellan/all.tsv","all.tsv",6.0947802E7]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"java.io.FileNotFoundException: /datasets/magellan","error":"<div class=\"ansiout\">\tat com.databricks.backend.daemon.data.client.DbfsClient.send0(DbfsClient.scala:63)\n\tat com.databricks.backend.daemon.data.client.DbfsClient.sendIdempotent(DbfsClient.scala:40)\n\tat com.databricks.backend.daemon.data.client.DatabricksFileSystem.listStatus(DatabricksFileSystem.scala:174)\n\tat com.databricks.backend.daemon.dbutils.FSUtils$.ls(DBUtilsCore.scala:60)\n\tat com.databricks.dbutils_v1.package$fs$.ls(dbutils_v1.scala:44)</div>","startTime":1.463634041158E12,"submitTime":1.463633885932E12,"finishTime":1.463634041569E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"80faa080-cc6f-49cd-9b9a-6c759b4c110a"},{"version":"CommandV1","origId":116372,"guid":"57ccafb4-6442-4de9-b2cb-64427159283b","subtype":"command","commandType":"auto","position":13.265625,"command":"%md \nFirst five lines or rows of the uber data containing: tripID, timestamp, Lon, Lat","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451993664E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"51c99ddd-7c45-45ea-ae92-053c843bd685"},{"version":"CommandV1","origId":116373,"guid":"038654c9-2ef5-4739-af14-213d6751753b","subtype":"command","commandType":"auto","position":13.28125,"command":"sc.textFile(\"dbfs:/datasets/magellan/all.tsv\").take(5).foreach(println)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">00001\t2007-01-07T10:54:50+00:00\t37.782551\t-122.445368\n00001\t2007-01-07T10:54:54+00:00\t37.782745\t-122.444586\n00001\t2007-01-07T10:54:58+00:00\t37.782842\t-122.443688\n00001\t2007-01-07T10:55:02+00:00\t37.782919\t-122.442815\n00001\t2007-01-07T10:55:06+00:00\t37.782992\t-122.442112\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: dbfs:/datasets/magellan/all.tsv","error":"<div class=\"ansiout\">\tat org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:197)\n\tat org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:208)\n\tat org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:207)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:237)\n\tat org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:237)\n\tat org.apache.spark.rdd.RDD$$anonfun$take$1.apply(RDD.scala:1281)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)\n\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:310)\n\tat org.apache.spark.rdd.RDD.take(RDD.scala:1276)</div>","startTime":1.463634065505E12,"submitTime":1.463633910272E12,"finishTime":1.46363406652E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"fb943ca7-a5b3-4e8d-909c-421b81c286f6"},{"version":"CommandV1","origId":116374,"guid":"b4b43486-90ad-419b-aaeb-de08a2cc9dee","subtype":"command","commandType":"auto","position":13.3125,"command":"display(dbutils.fs.ls(\"dbfs:/datasets/magellan/SFNbhd\")) // legacy shape files","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.dbf","planning_neighborhoods.dbf",1028.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.prj","planning_neighborhoods.prj",567.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.sbn","planning_neighborhoods.sbn",516.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.sbx","planning_neighborhoods.sbx",164.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.shp","planning_neighborhoods.shp",214576.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.shp.xml","planning_neighborhoods.shp.xml",21958.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.shx","planning_neighborhoods.shx",396.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"java.io.FileNotFoundException: /datasets/magellan/SFNbhd","error":"<div class=\"ansiout\">\tat com.databricks.backend.daemon.data.client.DbfsClient.send0(DbfsClient.scala:63)\n\tat com.databricks.backend.daemon.data.client.DbfsClient.sendIdempotent(DbfsClient.scala:40)\n\tat com.databricks.backend.daemon.data.client.DatabricksFileSystem.listStatus(DatabricksFileSystem.scala:174)\n\tat com.databricks.backend.daemon.dbutils.FSUtils$.ls(DBUtilsCore.scala:60)\n\tat com.databricks.dbutils_v1.package$fs$.ls(dbutils_v1.scala:44)</div>","startTime":1.46363409016E12,"submitTime":1.463633934936E12,"finishTime":1.463634090614E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"808057d7-681f-49c3-8989-c7437650aece"},{"version":"CommandV1","origId":116414,"guid":"ca114963-e39f-4583-83f8-a190ae7b64f1","subtype":"command","commandType":"auto","position":13.4287109375,"command":"%md \n#### Homework\n\nFirst watch the more technical magellan presentation by Ram Sri Harsha (Hortonworks) in Spark Summit Europe 2015\n\n[![Ram Sri Harsha's Magellan Spark Summit EU 2015 Talk]](http://img.youtube.com/vi/rP8H-xQTuM0/0.jpg)](https://www.youtube.com/watch?v=rP8H-xQTuM0)\n  \n\nSecond, carefully repeat Ram's original analysis from the following blog as done below.\n\n[Ram's blog in HortonWorks](http://hortonworks.com/blog/magellan-geospatial-analytics-in-spark/) and the [ZeppelinHub view of the demo code in video above](https://www.zeppelinhub.com/viewer/notebooks/aHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL2hvcnRvbndvcmtzLWdhbGxlcnkvemVwcGVsaW4tbm90ZWJvb2tzL21hc3Rlci8yQjRUV0dDOE0vbm90ZS5qc29u)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451996101E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cadc58a1-b039-432d-83f1-ac557ce60f33"},{"version":"CommandV1","origId":116415,"guid":"bac48345-045b-46cf-9c61-6e66321a15c2","subtype":"command","commandType":"auto","position":13.46435546875,"command":"case class UberRecord(tripId: String, timestamp: String, point: Point) // a case class for UberRecord ","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">defined class UberRecord\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.46363414299E12,"submitTime":1.463633987777E12,"finishTime":1.463634143081E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"817afdbd-0196-4f24-8c3c-1927f884fbdc"},{"version":"CommandV1","origId":116416,"guid":"3e67efc4-9b8f-4977-b831-b41fe528f88e","subtype":"command","commandType":"auto","position":13.5,"command":"val uber = sc.textFile(\"dbfs:/datasets/magellan/all.tsv\")\n              .map { line =>\n                      val parts = line.split(\"\\t\" )\n                      val tripId = parts(0)\n                      val timestamp = parts(1)\n                      val point = Point(parts(3).toDouble, parts(2).toDouble)\n                      UberRecord(tripId, timestamp, point)\n                    }.repartition(100)\n                     .toDF()\n                     .cache()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">uber: org.apache.spark.sql.DataFrame = [tripId: string, timestamp: string, point: poin]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\">&lt;console&gt;:47: error: not enough arguments for method apply: (tripId: Int, timestamp: java.sql.Timestamp, point: magellan.Point, time: Long)UberRecord in object UberRecord.\nUnspecified value parameter time.\n       UberRecord(tripId, timestamp, point)\n                 ^\n</div>","error":null,"startTime":1.463634181298E12,"submitTime":1.463634026059E12,"finishTime":1.463634182437E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b0940c12-09f1-492d-88b7-d30bd2a1846c"},{"version":"CommandV1","origId":116417,"guid":"45bf3401-ddcf-40a2-8b9a-01e0f4bbaa42","subtype":"command","commandType":"auto","position":13.625,"command":"val uberRecordCount = uber.count() // how many Uber records?","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">uberRecordCount: Long = 1128663\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634189853E12,"submitTime":1.46363403463E12,"finishTime":1.463634200608E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9bb5eafa-e5a6-4d83-b8a0-16f01fa9362f"},{"version":"CommandV1","origId":116418,"guid":"4aeed8c6-e468-4b08-b2fb-fe8e31e8be15","subtype":"command","commandType":"auto","position":13.6875,"command":"%md \nSo there are over a million ```UberRecord```s.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451996227E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"72e474b9-af02-49ed-889e-5a72d99f4763"},{"version":"CommandV1","origId":116419,"guid":"a35c0f86-f4be-4ac5-a68a-80582898e4de","subtype":"command","commandType":"auto","position":13.75,"command":"val neighborhoods = sqlContext.read.format(\"magellan\")\n                                   .load(\"dbfs:/datasets/magellan/SFNbhd/\")\n                                   .select($\"polygon\", $\"metadata\")\n                                   .cache()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">neighborhoods: org.apache.spark.sql.DataFrame = [polygon: pol, metadata: map&lt;string,string&gt;]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634218563E12,"submitTime":1.463634063342E12,"finishTime":1.463634218642E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c472c0a8-ffbd-46ea-9344-34047e4d30c7"},{"version":"CommandV1","origId":116420,"guid":"7f742272-aa52-4b27-b83b-af07d62094b2","subtype":"command","commandType":"auto","position":13.875,"command":"neighborhoods.count() // how many neighbourhoods in SF?","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res10: Long = 37\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634223143E12,"submitTime":1.463634067914E12,"finishTime":1.463634223368E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"ec47f963-ca3c-4d2c-8b7a-4e9c631207e6"},{"version":"CommandV1","origId":116421,"guid":"33c6d06e-d6bf-446d-ab1e-ea6abfa869e1","subtype":"command","commandType":"auto","position":13.9140625,"command":"neighborhoods.printSchema","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">root\n |-- polygon: pol (nullable = true)\n |-- metadata: map (nullable = true)\n |    |-- key: string\n |    |-- value: string (valueContainsNull = true)\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634228193E12,"submitTime":1.463634072964E12,"finishTime":1.463634228267E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"de89c69e-620f-4c82-be00-d4091af1c467"},{"version":"CommandV1","origId":116422,"guid":"bf37790b-3c1f-42d9-8670-aa53191d6375","subtype":"command","commandType":"auto","position":13.921875,"command":"neighborhoods.take(2) // see the first two neighbourhoods","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res12: Array[org.apache.spark.sql.Row] = Array([Polygon(WrappedArray(0), ArrayBuffer(Point(5994636.906492889, 2102065.0593099594), Point(5994947.421947971, 2102168.5643523037), Point(5995373.739008725, 2102188.865164712), Point(5995597.047945306, 2102107.661915049), Point(5995962.462568805, 2101985.8570405543), Point(5996145.169880569, 2102087.3611026257), Point(5996185.771505401, 2102270.068414375), Point(5996226.373130232, 2102574.580600634), Point(5996368.47881715, 2102797.8895372152), Point(5996449.682066813, 2103061.800098628), Point(5996551.186128899, 2103467.8163469583), Point(5996794.795877889, 2103853.5317828804), Point(5996794.795877889, 2104097.1415318847), Point(5996997.8040020615, 2104279.848843634), Point(5997200.812126234, 2104421.954858631), Point(5997586.527562141, 2104543.7597331256), Point(5997577.315638319, 2104615.921334222), Point(5998073.747388229, 2104665.5642795414), Point(5998500.0644489825, 2104726.467044875), Point(5998723.3733855635, 2104827.9711069614), Point(5998804.576635227, 2104949.775981471), Point(5998885.77988489, 2105112.182480797), Point(5998926.381509721, 2105254.288167715), Point(5999039.185089901, 2105423.4935379624), Point(5999046.57385464, 2105417.2415819615), Point(5999046.57385464, 2105297.722464055), Point(5999066.493762314, 2104939.164782211), Point(5999006.7343674, 2104640.367151469), Point(5999126.253485307, 2104620.447243795), Point(5999464.890931398, 2104421.248495221), Point(5999564.490141645, 2104401.328915626), Point(5999664.089679971, 2104441.1684028804), Point(5999843.368192807, 2104321.649284959), Point(5999982.807218388, 2104182.210259378), Point(6000122.2462439835, 2103943.1720235497), Point(6000241.76536189, 2103803.732997969), Point(6000401.124295145, 2103783.8130902946), Point(6000561.695496306, 2103641.083060801), Point(6000580.40313606, 2103624.4541570544), Point(6000520.643413067, 2103425.2557365447), Point(6000361.284479812, 2103106.5378700495), Point(6000162.086059317, 2102827.6601469666), Point(6000042.566941395, 2102429.262977883), Point(5999903.1279158145, 2101732.067849964), Point(5999763.688890234, 2101094.6324450523), Point(5999584.410049304, 2100815.7543938756), Point(5999604.329956979, 2100616.555973381), Point(5999604.329956979, 2100397.437317133), Point(5999444.9710237235, 2100317.758014545), Point(5999414.325087652, 2100274.8535728008), Point(5999405.131536484, 2100297.8381068856), Point(5998548.577475309, 2100138.4791736305), Point(5997692.023742229, 2099899.440937802), Point(5997399.803198054, 2099782.5527201295), Point(5997353.386296138, 2099839.6815428883), Point(5997194.027362898, 2099859.6014505476), Point(5996795.630193815, 2099879.521030128), Point(5996516.752142638, 2100058.7998710424), Point(5996078.515158221, 2100198.238896638), Point(5995600.438686565, 2100337.677922219), Point(5995401.239937976, 2100277.9181992114), Point(5995281.720820054, 2100497.036855459), Point(5995381.320358396, 2100676.3153682947), Point(5995381.320358396, 2100795.8344862163), Point(5995102.442307234, 2101154.392168045), Point(5994843.484163731, 2101493.0296141356), Point(5994803.644348398, 2101652.388547376), Point(5994644.285415143, 2102010.9459011257), Point(5994636.906492889, 2102065.0593099594))),Map(neighborho -&gt; Twin Peaks               )], [Polygon(WrappedArray(0), ArrayBuffer(Point(5999125.909982055, 2114969.0919007957), Point(5999027.885571808, 2115851.312905386), Point(5998919.275552809, 2116575.3780582994), Point(5999332.397758067, 2116643.0445896387), Point(5999218.223445728, 2117694.6494417936), Point(5999541.0098736435, 2117736.7520478815), Point(6005849.167860985, 2118601.3835207075), Point(6005946.912416145, 2117913.4127838016), Point(6006133.170245722, 2116602.4429647923), Point(6006224.980757639, 2115951.0259605497), Point(5999439.0199201405, 2115021.276835799), Point(5999125.909982055, 2114969.0919007957))),Map(neighborho -&gt; Pacific Heights          )])\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634239434E12,"submitTime":1.463634084206E12,"finishTime":1.463634239545E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c02bde69-472b-4d53-b7fc-803ae3831edb"},{"version":"CommandV1","origId":116423,"guid":"474a932d-eb5b-4c00-bb4b-7e6353948187","subtype":"command","commandType":"auto","position":13.931640625,"command":"neighborhoods.select(explode($\"metadata\").as(Seq(\"k\", \"v\"))).show(5,false)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+----------+-------------------------+\n|k         |v                        |\n+----------+-------------------------+\n|neighborho|Twin Peaks               |\n|neighborho|Pacific Heights          |\n|neighborho|Visitacion Valley        |\n|neighborho|Potrero Hill             |\n|neighborho|Crocker Amazon           |\n+----------+-------------------------+\nonly showing top 5 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634267166E12,"submitTime":1.463634111939E12,"finishTime":1.463634267322E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"e62aeb9d-d947-48f3-b07a-5c7baa910382"},{"version":"CommandV1","origId":118514,"guid":"88f671b1-2841-45c5-9e12-12fb2d5b32b1","subtype":"command","commandType":"auto","position":13.931884765625,"command":"%md\nThis join yields nothing. \n\nSo what's going on?\n\nWatch Ram's 2015 Spark Summit talk for details on geospatial formats and transformations.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7c6af768-b339-4dec-81f4-934536ae6a27"},{"version":"CommandV1","origId":116424,"guid":"8b11e2d1-cbde-49f6-9867-109fbaca0080","subtype":"command","commandType":"auto","position":13.93212890625,"command":"neighborhoods\n  .join(uber)\n  .where($\"point\" within $\"polygon\")\n  .select($\"tripId\", $\"timestamp\", explode($\"metadata\").as(Seq(\"k\", \"v\")))\n  .withColumnRenamed(\"v\", \"neighborhood\")\n  .drop(\"k\")\n  .show(5)\n","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+------+---------+------------+\n|tripId|timestamp|neighborhood|\n+------+---------+------------+\n+------+---------+------------+\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.4636342824E12,"submitTime":1.463634127165E12,"finishTime":1.463634363434E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cba85fe5-6217-4381-91c3-fe077691eb8f"},{"version":"CommandV1","origId":118515,"guid":"a060a280-0aea-4692-9ba1-3feb735bcf03","subtype":"command","commandType":"auto","position":13.9322509765625,"command":"%md\nNeed the right `transformer` to transform the points into the right coordinate system of the shape files.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"45a394a6-3c96-4e56-91b9-4e6e5a09d83a"},{"version":"CommandV1","origId":116425,"guid":"97af417c-ad63-44a5-8fbf-180d304e1173","subtype":"command","commandType":"auto","position":13.932373046875,"command":"val transformer: Point => Point = (point: Point) => \n{\n    val from = new NAD83(Map(\"zone\" -> 403)).from()\n    val p = point.transform(from)\n    new Point(3.28084 * p.x, 3.28084 * p.y)\n}\n\n// add a new column in nad83 coordinates\nval uberTransformed = uber\n                      .withColumn(\"nad83\", $\"point\".transform(transformer))\n                      .cache()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">transformer: magellan.Point =&gt; magellan.Point = &lt;function1&gt;\nuberTransformed: org.apache.spark.sql.DataFrame = [tripId: string, timestamp: string, point: poin, nad83: poin]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634380624E12,"submitTime":1.463634225382E12,"finishTime":1.46363438082E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"4d40dd65-3063-49dd-b87e-4c531657d5c1"},{"version":"CommandV1","origId":118516,"guid":"57b87530-03c9-4b12-8d22-d2f316759960","subtype":"command","commandType":"auto","position":13.9324951171875,"command":"%md\nLet' try the join again after appropriate transformation of coordinate system.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"15c11eca-4c06-4984-860b-5e23a06ce84e"},{"version":"CommandV1","origId":116426,"guid":"0cf7410b-4a61-47df-93ef-35a822e3022b","subtype":"command","commandType":"auto","position":13.9326171875,"command":"val joined = neighborhoods\n              .join(uberTransformed)\n              .where($\"nad83\" within $\"polygon\")\n              .select($\"tripId\", $\"timestamp\", explode($\"metadata\").as(Seq(\"k\", \"v\")))\n              .withColumnRenamed(\"v\", \"neighborhood\")\n              .drop(\"k\")\n              .cache()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">joined: org.apache.spark.sql.DataFrame = [tripId: string, timestamp: string, neighborhood: string]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634408281E12,"submitTime":1.463634253051E12,"finishTime":1.463634408493E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"a4daa35f-1e21-4057-8828-9a3fe0e50e30"},{"version":"CommandV1","origId":116427,"guid":"4eb40409-c2e5-48df-9de0-0579849e397f","subtype":"command","commandType":"auto","position":13.932861328125,"command":"joined.show(5,false)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+------+-------------------------+-------------------------+\n|tripId|timestamp                |neighborhood             |\n+------+-------------------------+-------------------------+\n|13375 |2007-01-04T03:15:30+00:00|Twin Peaks               |\n|13474 |2007-01-06T09:55:15+00:00|Twin Peaks               |\n|14110 |2007-01-05T03:07:37+00:00|Twin Peaks               |\n|15243 |2007-01-06T02:20:51+00:00|Twin Peaks               |\n|15603 |2007-01-06T03:41:24+00:00|Twin Peaks               |\n+------+-------------------------+-------------------------+\nonly showing top 5 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634415324E12,"submitTime":1.463634260096E12,"finishTime":1.463634423322E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"787301f4-d295-441b-ba31-8b14bf19ddd2"},{"version":"CommandV1","origId":118517,"guid":"7d4da486-a44a-42d5-bd79-a767c520ce53","subtype":"command","commandType":"auto","position":13.9329833984375,"command":"val UberRecordsInNbhdsCount = joined.count() // about 131 seconds for first action (doing broadcast hash join)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">UberRecordsInNbhdsCount: Long = 1085012\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463634426912E12,"submitTime":1.463634271682E12,"finishTime":1.463634484885E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"4bd2c7c6-23f7-4dfc-87d2-ad72483be607"},{"version":"CommandV1","origId":118518,"guid":"e50337fe-d848-4870-96c8-6569bea33510","subtype":"command","commandType":"auto","position":13.93304443359375,"command":"uberRecordCount - UberRecordsInNbhdsCount // records not in the neighbouthood shape files","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res16: Long = 43651\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463634490691E12,"submitTime":1.463634335452E12,"finishTime":1.463634491067E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"027b75e5-b4ec-4612-b3d5-570b8ba252e4"},{"version":"CommandV1","origId":116428,"guid":"7c277831-13ec-44dd-8dd2-d0f8dd06f3c1","subtype":"command","commandType":"auto","position":13.93310546875,"command":"joined\n  .groupBy($\"neighborhood\")\n  .agg(countDistinct(\"tripId\")\n  .as(\"trips\"))\n  .orderBy(col(\"trips\").desc)\n  .show(5,false)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+-------------------------+-----+\n|neighborhood             |trips|\n+-------------------------+-----+\n|South of Market          |9891 |\n|Western Addition         |6794 |\n|Downtown/Civic Center    |6697 |\n|Financial District       |6038 |\n|Mission                  |5620 |\n+-------------------------+-----+\nonly showing top 5 rows\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463634493301E12,"submitTime":1.46363433804E12,"finishTime":1.463634507626E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"394f620f-723d-49f1-a7b3-8b85ad40e476"},{"version":"CommandV1","origId":116429,"guid":"79c5d000-7312-4908-a64c-11bd41d397e8","subtype":"command","commandType":"auto","position":13.966552734375,"command":"%md\n## Spatio-temporal Queries \n\ncan be expressed in SQL using the Boolean predicates such as, \\\\(\\in , \\cap, \\ldots \\\\), that operate over space-time sets given products of 2D magellan objects and 1D time intervals.\n\nWant to scalably do the following:\n* Given :\n  * a set of trajectories as labelled points in space-time and \n  * a product of a time interval [ts,te] and a polygon P\n* Find all labelled space-time points that satisfy the following relations:\n    * intersect with [ts,te] X P\n    * the start-time of the ride or the end time of the ride intersects with [ts,te] X P\n    * intersect within a given distance d of any point or a given point in P (optional)\n    \nThis will allow us to answer questions like:\n* Where did the passengers who were using Uber and present in the SoMa neighbourhood in a given time interval get off?\n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451996701E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"db4de0df-8e4f-43b0-8a7f-a2cf8c394ff6"},{"version":"CommandV1","origId":116430,"guid":"0f0115b3-bf09-48a8-9ce4-874573486c91","subtype":"command","commandType":"auto","position":14.0,"command":"%md\n## Other spatial Algorithms in Spark are being explored for generic and more efficient scalable geospatial analytic tasks\nSee the Spark Summit East 2016 Talk by Ram on \"what next?\"\n\n* [SpatialSpark](http://spark-packages.org/package/syoummer/SpatialSpark) aims to provide efficient spatial operations using Apache Spark.\n  * Spatial Partition \n      * Generate a spatial partition from input dataset, currently Fixed-Grid Partition (FGP), Binary-Split Partition (BSP) and Sort-Tile Partition (STP) are supported. \n  * Spatial Range Query\n      *  includes both indexed and non-indexed query (useful for neighbourhood searches)\n* [z-order Knn join](https://github.com/anantasty/SparkAlgorithms/tree/master/mllib/src/main/scala/org/sparkalgos/mllib/join)\n  * A space-filling curve trick to index multi-dimensional metric data into 1 Dimension. See: [ieee paper](http://ieeexplore.ieee.org.ezproxy.canterbury.ac.nz/stamp/stamp.jsp?tp=&arnumber=5447837) and the [slides](http://www.slideshare.net/AshutoshTrivedi3/spark-algorithms).\n  \n* AkNN = All K Nearest Neighbours - identify the k nearesy neighbours for all nodes **simultaneously** (cont AkNN is the streaming form of AkNN)\n  * need to identify the right resources to do this scalably.\n* spark-knn-graphs: [https://github.com/tdebatty/spark-knn-graphs](https://github.com/tdebatty/spark-knn-graphs)\n***\n***","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451996764E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9dfaa8ac-4dad-4ce9-8005-815978cc0646"},{"version":"CommandV1","origId":116431,"guid":"60be54be-81dd-487a-b914-c9796c725c23","subtype":"command","commandType":"auto","position":14.5,"command":"%md \n# Downloading datasets and putting them in distributed file system","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451996829E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"5907f717-3a64-4c40-b23b-0c59f0c6ee92"},{"version":"CommandV1","origId":116432,"guid":"ed4c69b2-ea39-46a9-885e-19092a3c8992","subtype":"command","commandType":"auto","position":14.625,"command":"%md\n## getting uber data \n### (This only needs to be done once per shard!)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451996895E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9a8770f0-8cec-450d-84c7-25ef48ebd91e"},{"version":"CommandV1","origId":116433,"guid":"45772cbb-6f7e-40b1-b16f-9944a87a1177","subtype":"command","commandType":"auto","position":14.65625,"command":"%sh ls","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">eventlogs\nlogs\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Cancelled","error":null,"startTime":1.463618753235E12,"submitTime":1.463618596988E12,"finishTime":1.463618753557E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"df67dfcd-6d94-4ee2-8135-2c77789e2b4c"},{"version":"CommandV1","origId":116434,"guid":"b6f93e29-8975-4131-8ec5-3916c56459e3","subtype":"command","commandType":"auto","position":14.6875,"command":"%sh\nwget https://raw.githubusercontent.com/dima42/uber-gps-analysis/master/gpsdata/all.tsv","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">--2016-05-19 00:45:55--  https://raw.githubusercontent.com/dima42/uber-gps-analysis/master/gpsdata/all.tsv\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 103.245.222.133\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|103.245.222.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 60947802 (58M) [text/plain]\nSaving to: &apos;all.tsv&apos;\n\n     0K .......... .......... .......... .......... ..........  0%  728K 82s\n    50K .......... .......... .......... .......... ..........  0% 1.43M 61s\n   100K .......... .......... .......... .......... ..........  0% 47.9M 41s\n   150K .......... .......... .......... .......... ..........  0%  145M 31s\n   200K .......... .......... .......... .......... ..........  0% 1.45M 33s\n   250K .......... .......... .......... .......... ..........  0% 47.4M 27s\n   300K .......... .......... .......... .......... ..........  0%  162M 24s\n   350K .......... .......... .......... .......... ..........  0%  181M 21s\n   400K .......... .......... .......... .......... ..........  0% 1.50M 23s\n   450K .......... .......... .......... .......... ..........  0% 26.7M 20s\n   500K .......... .......... .......... .......... ..........  0%  208M 19s\n   550K .......... .......... .......... .......... ..........  1%  192M 17s\n   600K .......... .......... .......... .......... ..........  1%  186M 16s\n   650K .......... .......... .......... .......... ..........  1%  206M 15s\n   700K .......... .......... .......... .......... ..........  1%  210M 14s\n   750K .......... .......... .......... .......... ..........  1%  175M 13s\n   800K .......... .......... .......... .......... ..........  1% 1.56M 14s\n   850K .......... .......... .......... .......... ..........  1% 29.0M 14s\n   900K .......... .......... .......... .......... ..........  1%  190M 13s\n   950K .......... .......... .......... .......... ..........  1%  181M 12s\n  1000K .......... .......... .......... .......... ..........  1%  216M 12s\n  1050K .......... .......... .......... .......... ..........  1%  184M 11s\n  1100K .......... .......... .......... .......... ..........  1%  212M 11s\n  1150K .......... .......... .......... .......... ..........  2% 1.61M 12s\n  1200K .......... .......... .......... .......... ..........  2% 33.1M 11s\n  1250K .......... .......... .......... .......... ..........  2% 36.8M 11s\n  1300K .......... .......... .......... .......... ..........  2%  211M 10s\n  1350K .......... .......... .......... .......... ..........  2%  173M 10s\n  1400K .......... .......... .......... .......... ..........  2%  209M 10s\n  1450K .......... .......... .......... .......... ..........  2%  205M 9s\n  1500K .......... .......... .......... .......... ..........  2%  216M 9s\n  1550K .......... .......... .......... .......... ..........  2% 1.59M 10s\n  1600K .......... .......... .......... .......... ..........  2% 59.8M 10s\n  1650K .......... .......... .......... .......... ..........  2% 75.2M 9s\n  1700K .......... .......... .......... .......... ..........  2% 38.6M 9s\n  1750K .......... .......... .......... .......... ..........  3%  185M 9s\n  1800K .......... .......... .......... .......... ..........  3%  211M 9s\n  1850K .......... .......... .......... .......... ..........  3%  189M 8s\n  1900K .......... .......... .......... .......... ..........  3%  211M 8s\n  1950K .......... .......... .......... .......... ..........  3% 1.59M 9s\n  2000K .......... .......... .......... .......... ..........  3% 59.9M 9s\n  2050K .......... .......... .......... .......... ..........  3% 40.5M 9s\n  2100K .......... .......... .......... .......... ..........  3% 62.4M 8s\n  2150K .......... .......... .......... .......... ..........  3%  175M 8s\n  2200K .......... .......... .......... .......... ..........  3%  211M 8s\n  2250K .......... .......... .......... .......... ..........  3%  212M 8s\n  2300K .......... .......... .......... .......... ..........  3%  213M 8s\n  2350K .......... .......... .......... .......... ..........  4% 1.56M 8s\n  2400K .......... .......... .......... .......... ..........  4%  186M 8s\n  2450K .......... .......... .......... .......... ..........  4% 43.8M 8s\n  2500K .......... .......... .......... .......... ..........  4% 71.9M 8s\n  2550K .......... .......... .......... .......... ..........  4%  182M 8s\n  2600K .......... .......... .......... .......... ..........  4%  209M 7s\n  2650K .......... .......... .......... .......... ..........  4%  209M 7s\n  2700K .......... .......... .......... .......... ..........  4% 1.63M 8s\n  2750K .......... .......... .......... .......... ..........  4% 27.7M 8s\n  2800K .......... .......... .......... .......... ..........  4%  138M 8s\n  2850K .......... .......... .......... .......... ..........  4% 39.2M 7s\n  2900K .......... .......... .......... .......... ..........  4%  103M 7s\n  2950K .......... .......... .......... .......... ..........  5%  180M 7s\n  3000K .......... .......... .......... .......... ..........  5%  162M 7s\n  3050K .......... .......... .......... .......... ..........  5%  212M 7s\n  3100K .......... .......... .......... .......... ..........  5% 1.61M 7s\n  3150K .......... .......... .......... .......... ..........  5% 35.7M 7s\n  3200K .......... .......... .......... .......... ..........  5% 63.6M 7s\n  3250K .......... .......... .......... .......... ..........  5% 54.0M 7s\n  3300K .......... .......... .......... .......... ..........  5%  114M 7s\n  3350K .......... .......... .......... .......... ..........  5%  182M 7s\n  3400K .......... .......... .......... .......... ..........  5%  210M 7s\n  3450K .......... .......... .......... .......... ..........  5%  207M 7s\n  3500K .......... .......... .......... .......... ..........  5% 1.59M 7s\n  3550K .......... .......... .......... .......... ..........  6% 14.8M 7s\n  3600K .......... .......... .......... .......... ..........  6%  114M 7s\n  3650K .......... .......... .......... .......... ..........  6%  111M 7s\n  3700K .......... .......... .......... .......... ..........  6%  117M 7s\n  3750K .......... .......... .......... .......... ..........  6%  102M 7s\n  3800K .......... .......... .......... .......... ..........  6%  115M 7s\n  3850K .......... .......... .......... .......... ..........  6% 1.76M 7s\n  3900K .......... .......... .......... .......... ..........  6% 19.6M 7s\n  3950K .......... .......... .......... .......... ..........  6% 82.4M 7s\n  4000K .......... .......... .......... .......... ..........  6%  174M 7s\n  4050K .......... .......... .......... .......... ..........  6% 42.8M 7s\n  4100K .......... .......... .......... .......... ..........  6%  209M 6s\n  4150K .......... .......... .......... .......... ..........  7%  181M 6s\n  4200K .......... .......... .......... .......... ..........  7%  208M 6s\n  4250K .......... .......... .......... .......... ..........  7% 1.65M 7s\n  4300K .......... .......... .......... .......... ..........  7% 24.5M 7s\n  4350K .......... .......... .......... .......... ..........  7%  107M 6s\n  4400K .......... .......... .......... .......... ..........  7% 93.0M 6s\n  4450K .......... .......... .......... .......... ..........  7% 43.3M 6s\n  4500K .......... .......... .......... .......... ..........  7%  208M 6s\n  4550K .......... .......... .......... .......... ..........  7%  185M 6s\n  4600K .......... .......... .......... .......... ..........  7%  211M 6s\n  4650K .......... .......... .......... .......... ..........  7% 1.61M 6s\n  4700K .......... .......... .......... .......... ..........  7% 33.3M 6s\n  4750K .......... .......... .......... .......... ..........  8%  127M 6s\n  4800K .......... .......... .......... .......... ..........  8%  114M 6s\n  4850K .......... .......... .......... .......... ..........  8% 66.2M 6s\n  4900K .......... .......... .......... .......... ..........  8% 68.0M 6s\n  4950K .......... .......... .......... .......... ..........  8%  154M 6s\n  5000K .......... .......... .......... .......... ..........  8% 1.69M 6s\n  5050K .......... .......... .......... .......... ..........  8% 16.7M 6s\n  5100K .......... .......... .......... .......... ..........  8%  180M 6s\n  5150K .......... .......... .......... .......... ..........  8% 89.3M 6s\n  5200K .......... .......... .......... .......... ..........  8% 67.5M 6s\n  5250K .......... .......... .......... .......... ..........  8% 69.5M 6s\n  5300K .......... .......... .......... .......... ..........  8%  191M 6s\n  5350K .......... .......... .......... .......... ..........  9%  191M 6s\n  5400K .......... .......... .......... .......... ..........  9% 1.68M 6s\n  5450K .......... .......... .......... .......... ..........  9% 22.3M 6s\n  5500K .......... .......... .......... .......... ..........  9% 59.2M 6s\n  5550K .......... .......... .......... .......... ..........  9% 97.4M 6s\n  5600K .......... .......... .......... .......... ..........  9% 48.0M 6s\n  5650K .......... .......... .......... .......... ..........  9% 98.6M 6s\n  5700K .......... .......... .......... .......... ..........  9%  209M 6s\n  5750K .......... .......... .......... .......... ..........  9%  168M 6s\n  5800K .......... .......... .......... .......... ..........  9% 1.65M 6s\n  5850K .......... .......... .......... .......... ..........  9% 32.9M 6s\n  5900K .......... .......... .......... .......... ..........  9% 43.1M 6s\n  5950K .......... .......... .......... .......... .......... 10%  109M 6s\n  6000K .......... .......... .......... .......... .......... 10% 50.5M 6s\n  6050K .......... .......... .......... .......... .......... 10%  144M 6s\n  6100K .......... .......... .......... .......... .......... 10%  191M 6s\n  6150K .......... .......... .......... .......... .......... 10% 1.73M 6s\n  6200K .......... .......... .......... .......... .......... 10% 22.8M 6s\n  6250K .......... .......... .......... .......... .......... 10% 34.1M 6s\n  6300K .......... .......... .......... .......... .......... 10% 62.4M 6s\n  6350K .......... .......... .......... .......... .......... 10% 68.5M 6s\n  6400K .......... .......... .......... .......... .......... 10% 61.8M 6s\n  6450K .......... .......... .......... .......... .......... 10%  209M 6s\n  6500K .......... .......... .......... .......... .......... 11%  213M 6s\n  6550K .......... .......... .......... .......... .......... 11% 1.72M 6s\n  6600K .......... .......... .......... .......... .......... 11% 19.5M 6s\n  6650K .......... .......... .......... .......... .......... 11% 66.9M 6s\n  6700K .......... .......... .......... .......... .......... 11% 68.3M 6s\n  6750K .......... .......... .......... .......... .......... 11%  104M 6s\n  6800K .......... .......... .......... .......... .......... 11%  101M 6s\n  6850K .......... .......... .......... .......... .......... 11% 63.5M 6s\n  6900K .......... .......... .......... .......... .......... 11%  210M 5s\n  6950K .......... .......... .......... .......... .......... 11% 1.66M 6s\n  7000K .......... .......... .......... .......... .......... 11% 29.6M 6s\n  7050K .......... .......... .......... .......... .......... 11% 55.8M 6s\n  7100K .......... .......... .......... .......... .......... 12% 59.8M 6s\n  7150K .......... .......... .......... .......... .......... 12%  115M 6s\n  7200K .......... .......... .......... .......... .......... 12% 54.4M 5s\n  7250K .......... .......... .......... .......... .......... 12%  213M 5s\n  7300K .......... .......... .......... .......... .......... 12% 1.75M 6s\n  7350K .......... .......... .......... .......... .......... 12% 21.6M 6s\n  7400K .......... .......... .......... .......... .......... 12% 27.9M 6s\n  7450K .......... .......... .......... .......... .......... 12%  162M 5s\n  7500K .......... .......... .......... .......... .......... 12% 56.6M 5s\n  7550K .......... .......... .......... .......... .......... 12% 85.1M 5s\n  7600K .......... .......... .......... .......... .......... 12% 68.8M 5s\n  7650K .......... .......... .......... .......... .......... 12%  210M 5s\n  7700K .......... .......... .......... .......... .......... 13% 1.72M 5s\n  7750K .......... .......... .......... .......... .......... 13% 20.8M 5s\n  7800K .......... .......... .......... .......... .......... 13% 41.4M 5s\n  7850K .......... .......... .......... .......... .......... 13%  110M 5s\n  7900K .......... .......... .......... .......... .......... 13% 71.6M 5s\n  7950K .......... .......... .......... .......... .......... 13% 89.7M 5s\n  8000K .......... .......... .......... .......... .......... 13% 65.4M 5s\n  8050K .......... .......... .......... .......... .......... 13%  212M 5s\n  8100K .......... .......... .......... .......... .......... 13% 1.69M 5s\n  8150K .......... .......... .......... .......... .......... 13% 25.3M 5s\n  8200K .......... .......... .......... .......... .......... 13% 25.5M 5s\n  8250K .......... .......... .......... .......... .......... 13% 80.7M 5s\n  8300K .......... .......... .......... .......... .......... 14% 69.8M 5s\n  8350K .......... .......... .......... .......... .......... 14%  132M 5s\n  8400K .......... .......... .......... .......... .......... 14%  211M 5s\n  8450K .......... .......... .......... .......... .......... 14% 1.81M 5s\n  8500K .......... .......... .......... .......... .......... 14% 20.8M 5s\n  8550K .......... .......... .......... .......... .......... 14% 29.1M 5s\n  8600K .......... .......... .......... .......... .......... 14% 44.5M 5s\n  8650K .......... .......... .......... .......... .......... 14% 66.7M 5s\n  8700K .......... .......... .......... .......... .......... 14% 90.2M 5s\n  8750K .......... .......... .......... .......... .......... 14% 71.8M 5s\n  8800K .......... .......... .......... .......... .......... 14%  195M 5s\n  8850K .......... .......... .......... .......... .......... 14% 1.75M 5s\n  8900K .......... .......... .......... .......... .......... 15% 23.9M 5s\n  8950K .......... .......... .......... .......... .......... 15% 29.0M 5s\n  9000K .......... .......... .......... .......... .......... 15% 41.9M 5s\n  9050K .......... .......... .......... .......... .......... 15%  106M 5s\n  9100K .......... .......... .......... .......... .......... 15%  121M 5s\n  9150K .......... .......... .......... .......... .......... 15% 50.7M 5s\n  9200K .......... .......... .......... .......... .......... 15%  113M 5s\n  9250K .......... .......... .......... .......... .......... 15% 1.76M 5s\n  9300K .......... .......... .......... .......... .......... 15% 23.1M 5s\n  9350K .......... .......... .......... .......... .......... 15% 33.8M 5s\n  9400K .......... .......... .......... .......... .......... 15% 64.9M 5s\n  9450K .......... .......... .......... .......... .......... 15% 62.6M 5s\n  9500K .......... .......... .......... .......... .......... 16%  165M 5s\n  9550K .......... .......... .......... .......... .......... 16% 77.5M 5s\n  9600K .......... .......... .......... .......... .......... 16% 1.82M 5s\n  9650K .......... .......... .......... .......... .......... 16% 21.4M 5s\n  9700K .......... .......... .......... .......... .......... 16% 27.7M 5s\n  9750K .......... .......... .......... .......... .......... 16% 32.5M 5s\n  9800K .......... .......... .......... .......... .......... 16% 94.0M 5s\n  9850K .......... .......... .......... .......... .......... 16% 63.1M 5s\n  9900K .......... .......... .......... .......... .......... 16%  104M 5s\n  9950K .......... .......... .......... .......... .......... 16%  109M 5s\n 10000K .......... .......... .......... .......... .......... 16% 1.77M 5s\n 10050K .......... .......... .......... .......... .......... 16% 24.6M 5s\n 10100K .......... .......... .......... .......... .......... 17% 32.9M 5s\n 10150K .......... .......... .......... .......... .......... 17% 43.5M 5s\n 10200K .......... .......... .......... .......... .......... 17%  105M 5s\n 10250K .......... .......... .......... .......... .......... 17% 57.8M 5s\n 10300K .......... .......... .......... .......... .......... 17%  158M 5s\n 10350K .......... .......... .......... .......... .......... 17% 75.4M 5s\n 10400K .......... .......... .......... .......... .......... 17% 1.74M 5s\n 10450K .......... .......... .......... .......... .......... 17% 24.2M 5s\n 10500K .......... .......... .......... .......... .......... 17% 31.1M 5s\n 10550K .......... .......... .......... .......... .......... 17% 51.8M 5s\n 10600K .......... .......... .......... .......... .......... 17%  166M 5s\n 10650K .......... .......... .......... .......... .......... 17% 58.2M 5s\n 10700K .......... .......... .......... .......... .......... 18% 83.7M 5s\n 10750K .......... .......... .......... .......... .......... 18% 1.86M 5s\n 10800K .......... .......... .......... .......... .......... 18% 21.7M 5s\n 10850K .......... .......... .......... .......... .......... 18% 24.9M 5s\n 10900K .......... .......... .......... .......... .......... 18% 33.8M 5s\n 10950K .......... .......... .......... .......... .......... 18% 54.5M 5s\n 11000K .......... .......... .......... .......... .......... 18% 59.1M 5s\n 11050K .......... .......... .......... .......... .......... 18%  111M 5s\n 11100K .......... .......... .......... .......... .......... 18%  118M 5s\n 11150K .......... .......... .......... .......... .......... 18% 1.80M 5s\n 11200K .......... .......... .......... .......... .......... 18% 20.8M 5s\n 11250K .......... .......... .......... .......... .......... 18% 36.1M 5s\n 11300K .......... .......... .......... .......... .......... 19% 27.3M 5s\n 11350K .......... .......... .......... .......... .......... 19% 78.3M 5s\n 11400K .......... .......... .......... .......... .......... 19% 77.2M 5s\n 11450K .......... .......... .......... .......... .......... 19%  102M 5s\n 11500K .......... .......... .......... .......... .......... 19%  213M 5s\n 11550K .......... .......... .......... .......... .......... 19% 1.77M 5s\n 11600K .......... .......... .......... .......... .......... 19% 23.8M 5s\n 11650K .......... .......... .......... .......... .......... 19% 29.8M 5s\n 11700K .......... .......... .......... .......... .......... 19% 34.9M 5s\n 11750K .......... .......... .......... .......... .......... 19% 49.2M 5s\n 11800K .......... .......... .......... .......... .......... 19%  117M 5s\n 11850K .......... .......... .......... .......... .......... 19%  244M 5s\n 11900K .......... .......... .......... .......... .......... 20%  265M 5s\n 11950K .......... .......... .......... .......... .......... 20% 1.75M 5s\n 12000K .......... .......... .......... .......... .......... 20% 24.3M 5s\n 12050K .......... .......... .......... .......... .......... 20% 34.5M 5s\n 12100K .......... .......... .......... .......... .......... 20% 42.5M 5s\n 12150K .......... .......... .......... .......... .......... 20%  182M 5s\n 12200K .......... .......... .......... .......... .......... 20%  100M 5s\n 12250K .......... .......... .......... .......... .......... 20% 95.8M 5s\n 12300K .......... .......... .......... .......... .......... 20% 1.88M 5s\n 12350K .......... .......... .......... .......... .......... 20% 14.0M 5s\n 12400K .......... .......... .......... .......... .......... 20% 31.3M 5s\n 12450K .......... .......... .......... .......... .......... 21% 27.8M 5s\n 12500K .......... .......... .......... .......... .......... 21%  110M 5s\n 12550K .......... .......... .......... .......... .......... 21%  113M 5s\n 12600K .......... .......... .......... .......... .......... 21% 80.1M 5s\n 12650K .......... .......... .......... .......... .......... 21%  136M 5s\n 12700K .......... .......... .......... .......... .......... 21% 1.82M 5s\n 12750K .......... .......... .......... .......... .......... 21% 17.3M 5s\n 12800K .......... .......... .......... .......... .......... 21% 27.4M 5s\n 12850K .......... .......... .......... .......... .......... 21% 36.3M 5s\n 12900K .......... .......... .......... .......... .......... 21%  134M 5s\n 12950K .......... .......... .......... .......... .......... 21%  112M 5s\n 13000K .......... .......... .......... .......... .......... 21%  106M 5s\n 13050K .......... .......... .......... .......... .......... 22% 72.6M 5s\n 13100K .......... .......... .......... .......... .......... 22% 1.79M 5s\n 13150K .......... .......... .......... .......... .......... 22% 19.2M 5s\n 13200K .......... .......... .......... .......... .......... 22% 28.0M 5s\n 13250K .......... .......... .......... .......... .......... 22% 45.4M 5s\n 13300K .......... .......... .......... .......... .......... 22%  157M 5s\n 13350K .......... .......... .......... .......... .......... 22% 78.1M 5s\n 13400K .......... .......... .......... .......... .......... 22%  154M 4s\n 13450K .......... .......... .......... .......... .......... 22% 1.94M 5s\n 13500K .......... .......... .......... .......... .......... 22% 15.8M 5s\n 13550K .......... .......... .......... .......... .......... 22% 18.9M 5s\n 13600K .......... .......... .......... .......... .......... 22% 33.2M 5s\n 13650K .......... .......... .......... .......... .......... 23% 44.6M 4s\n 13700K .......... .......... .......... .......... .......... 23%  114M 4s\n 13750K .......... .......... .......... .......... .......... 23% 81.3M 4s\n 13800K .......... .......... .......... .......... .......... 23% 83.3M 4s\n 13850K .......... .......... .......... .......... .......... 23% 1.88M 5s\n 13900K .......... .......... .......... .......... .......... 23% 18.0M 4s\n 13950K .......... .......... .......... .......... .......... 23% 21.7M 4s\n 14000K .......... .......... .......... .......... .......... 23% 34.4M 4s\n 14050K .......... .......... .......... .......... .......... 23% 68.6M 4s\n 14100K .......... .......... .......... .......... .......... 23%  146M 4s\n 14150K .......... .......... .......... .......... .......... 23% 75.8M 4s\n 14200K .......... .......... .......... .......... .......... 23% 81.8M 4s\n 14250K .......... .......... .......... .......... .......... 24% 1.83M 4s\n 14300K .......... .......... .......... .......... .......... 24% 24.3M 4s\n 14350K .......... .......... .......... .......... .......... 24% 23.3M 4s\n 14400K .......... .......... .......... .......... .......... 24% 35.1M 4s\n 14450K .......... .......... .......... .......... .......... 24% 62.0M 4s\n 14500K .......... .......... .......... .......... .......... 24%  137M 4s\n 14550K .......... .......... .......... .......... .......... 24% 88.8M 4s\n 14600K .......... .......... .......... .......... .......... 24% 1.95M 4s\n 14650K .......... .......... .......... .......... .......... 24% 18.5M 4s\n 14700K .......... .......... .......... .......... .......... 24% 24.5M 4s\n 14750K .......... .......... .......... .......... .......... 24% 22.7M 4s\n 14800K .......... .......... .......... .......... .......... 24% 38.2M 4s\n 14850K .......... .......... .......... .......... .......... 25% 77.1M 4s\n 14900K .......... .......... .......... .......... .......... 25% 95.3M 4s\n 14950K .......... .......... .......... .......... .......... 25% 77.7M 4s\n 15000K .......... .......... .......... .......... .......... 25% 1.93M 4s\n 15050K .......... .......... .......... .......... .......... 25% 17.6M 4s\n 15100K .......... .......... .......... .......... .......... 25% 31.8M 4s\n 15150K .......... .......... .......... .......... .......... 25% 20.9M 4s\n 15200K .......... .......... .......... .......... .......... 25% 52.7M 4s\n 15250K .......... .......... .......... .......... .......... 25%  194M 4s\n 15300K .......... .......... .......... .......... .......... 25% 72.2M 4s\n 15350K .......... .......... .......... .......... .......... 25% 80.5M 4s\n 15400K .......... .......... .......... .......... .......... 25% 1.86M 4s\n 15450K .......... .......... .......... .......... .......... 26% 22.2M 4s\n 15500K .......... .......... .......... .......... .......... 26% 23.3M 4s\n 15550K .......... .......... .......... .......... .......... 26% 28.6M 4s\n 15600K .......... .......... .......... .......... .......... 26% 56.4M 4s\n 15650K .......... .......... .......... .......... .......... 26%  118M 4s\n 15700K .......... .......... .......... .......... .......... 26%  100M 4s\n 15750K .......... .......... .......... .......... .......... 26% 1.99M 4s\n 15800K .......... .......... .......... .......... .......... 26% 18.4M 4s\n 15850K .......... .......... .......... .......... .......... 26% 22.6M 4s\n 15900K .......... .......... .......... .......... .......... 26% 24.1M 4s\n 15950K .......... .......... .......... .......... .......... 26% 31.2M 4s\n 16000K .......... .......... .......... .......... .......... 26% 46.8M 4s\n 16050K .......... .......... .......... .......... .......... 27%  157M 4s\n 16100K .......... .......... .......... .......... .......... 27% 86.7M 4s\n 16150K .......... .......... .......... .......... .......... 27% 1.95M 4s\n... skipped 41040 bytes ...\n 43200K .......... .......... .......... .......... .......... 72% 22.4M 2s\n 43250K .......... .......... .......... .......... .......... 72% 37.4M 1s\n 43300K .......... .......... .......... .......... .......... 72% 27.1M 1s\n 43350K .......... .......... .......... .......... .......... 72% 34.8M 1s\n 43400K .......... .......... .......... .......... .......... 73% 41.2M 1s\n 43450K .......... .......... .......... .......... .......... 73% 2.52M 1s\n 43500K .......... .......... .......... .......... .......... 73% 10.9M 1s\n 43550K .......... .......... .......... .......... .......... 73% 15.9M 1s\n 43600K .......... .......... .......... .......... .......... 73% 21.7M 1s\n 43650K .......... .......... .......... .......... .......... 73% 43.2M 1s\n 43700K .......... .......... .......... .......... .......... 73% 20.6M 1s\n 43750K .......... .......... .......... .......... .......... 73% 48.2M 1s\n 43800K .......... .......... .......... .......... .......... 73% 3.15M 1s\n 43850K .......... .......... .......... .......... .......... 73% 9.45M 1s\n 43900K .......... .......... .......... .......... .......... 73% 9.12M 1s\n 43950K .......... .......... .......... .......... .......... 73% 21.5M 1s\n 44000K .......... .......... .......... .......... .......... 74% 26.9M 1s\n 44050K .......... .......... .......... .......... .......... 74% 39.5M 1s\n 44100K .......... .......... .......... .......... .......... 74% 22.3M 1s\n 44150K .......... .......... .......... .......... .......... 74% 45.8M 1s\n 44200K .......... .......... .......... .......... .......... 74% 2.89M 1s\n 44250K .......... .......... .......... .......... .......... 74% 9.69M 1s\n 44300K .......... .......... .......... .......... .......... 74% 10.1M 1s\n 44350K .......... .......... .......... .......... .......... 74% 24.4M 1s\n 44400K .......... .......... .......... .......... .......... 74% 32.6M 1s\n 44450K .......... .......... .......... .......... .......... 74% 32.8M 1s\n 44500K .......... .......... .......... .......... .......... 74% 24.5M 1s\n 44550K .......... .......... .......... .......... .......... 74% 40.7M 1s\n 44600K .......... .......... .......... .......... .......... 75% 2.59M 1s\n 44650K .......... .......... .......... .......... .......... 75% 13.4M 1s\n 44700K .......... .......... .......... .......... .......... 75% 11.4M 1s\n 44750K .......... .......... .......... .......... .......... 75% 23.9M 1s\n 44800K .......... .......... .......... .......... .......... 75% 35.9M 1s\n 44850K .......... .......... .......... .......... .......... 75% 23.7M 1s\n 44900K .......... .......... .......... .......... .......... 75% 38.1M 1s\n 44950K .......... .......... .......... .......... .......... 75% 3.25M 1s\n 45000K .......... .......... .......... .......... .......... 75% 8.60M 1s\n 45050K .......... .......... .......... .......... .......... 75% 13.0M 1s\n 45100K .......... .......... .......... .......... .......... 75% 13.4M 1s\n 45150K .......... .......... .......... .......... .......... 75% 21.8M 1s\n 45200K .......... .......... .......... .......... .......... 76% 46.9M 1s\n 45250K .......... .......... .......... .......... .......... 76% 23.2M 1s\n 45300K .......... .......... .......... .......... .......... 76% 47.5M 1s\n 45350K .......... .......... .......... .......... .......... 76% 2.92M 1s\n 45400K .......... .......... .......... .......... .......... 76% 9.67M 1s\n 45450K .......... .......... .......... .......... .......... 76% 13.2M 1s\n 45500K .......... .......... .......... .......... .......... 76% 14.2M 1s\n 45550K .......... .......... .......... .......... .......... 76% 32.4M 1s\n 45600K .......... .......... .......... .......... .......... 76% 36.2M 1s\n 45650K .......... .......... .......... .......... .......... 76% 26.7M 1s\n 45700K .......... .......... .......... .......... .......... 76% 34.1M 1s\n 45750K .......... .......... .......... .......... .......... 76% 2.63M 1s\n 45800K .......... .......... .......... .......... .......... 77% 12.6M 1s\n 45850K .......... .......... .......... .......... .......... 77% 13.9M 1s\n 45900K .......... .......... .......... .......... .......... 77% 16.9M 1s\n 45950K .......... .......... .......... .......... .......... 77% 36.4M 1s\n 46000K .......... .......... .......... .......... .......... 77% 24.8M 1s\n 46050K .......... .......... .......... .......... .......... 77% 36.7M 1s\n 46100K .......... .......... .......... .......... .......... 77% 3.17M 1s\n 46150K .......... .......... .......... .......... .......... 77% 9.36M 1s\n 46200K .......... .......... .......... .......... .......... 77% 12.6M 1s\n 46250K .......... .......... .......... .......... .......... 77% 14.6M 1s\n 46300K .......... .......... .......... .......... .......... 77% 19.9M 1s\n 46350K .......... .......... .......... .......... .......... 77% 39.9M 1s\n 46400K .......... .......... .......... .......... .......... 78% 22.6M 1s\n 46450K .......... .......... .......... .......... .......... 78% 51.2M 1s\n 46500K .......... .......... .......... .......... .......... 78% 2.97M 1s\n 46550K .......... .......... .......... .......... .......... 78% 9.82M 1s\n 46600K .......... .......... .......... .......... .......... 78% 12.7M 1s\n 46650K .......... .......... .......... .......... .......... 78% 16.8M 1s\n 46700K .......... .......... .......... .......... .......... 78% 22.1M 1s\n 46750K .......... .......... .......... .......... .......... 78% 33.9M 1s\n 46800K .......... .......... .......... .......... .......... 78% 28.1M 1s\n 46850K .......... .......... .......... .......... .......... 78% 35.8M 1s\n 46900K .......... .......... .......... .......... .......... 78% 2.67M 1s\n 46950K .......... .......... .......... .......... .......... 78% 12.1M 1s\n 47000K .......... .......... .......... .......... .......... 79% 14.0M 1s\n 47050K .......... .......... .......... .......... .......... 79% 24.3M 1s\n 47100K .......... .......... .......... .......... .......... 79% 21.2M 1s\n 47150K .......... .......... .......... .......... .......... 79% 24.7M 1s\n 47200K .......... .......... .......... .......... .......... 79% 32.3M 1s\n 47250K .......... .......... .......... .......... .......... 79% 3.35M 1s\n 47300K .......... .......... .......... .......... .......... 79% 8.97M 1s\n 47350K .......... .......... .......... .......... .......... 79% 11.5M 1s\n 47400K .......... .......... .......... .......... .......... 79% 14.8M 1s\n 47450K .......... .......... .......... .......... .......... 79% 23.7M 1s\n 47500K .......... .......... .......... .......... .......... 79% 24.3M 1s\n 47550K .......... .......... .......... .......... .......... 79% 26.1M 1s\n 47600K .......... .......... .......... .......... .......... 80% 41.0M 1s\n 47650K .......... .......... .......... .......... .......... 80% 3.05M 1s\n 47700K .......... .......... .......... .......... .......... 80% 9.77M 1s\n 47750K .......... .......... .......... .......... .......... 80% 12.5M 1s\n 47800K .......... .......... .......... .......... .......... 80% 15.0M 1s\n 47850K .......... .......... .......... .......... .......... 80% 29.9M 1s\n 47900K .......... .......... .......... .......... .......... 80% 28.6M 1s\n 47950K .......... .......... .......... .......... .......... 80% 24.0M 1s\n 48000K .......... .......... .......... .......... .......... 80% 44.1M 1s\n 48050K .......... .......... .......... .......... .......... 80% 2.69M 1s\n 48100K .......... .......... .......... .......... .......... 80% 12.2M 1s\n 48150K .......... .......... .......... .......... .......... 80% 14.1M 1s\n 48200K .......... .......... .......... .......... .......... 81% 19.6M 1s\n 48250K .......... .......... .......... .......... .......... 81% 22.5M 1s\n 48300K .......... .......... .......... .......... .......... 81% 25.7M 1s\n 48350K .......... .......... .......... .......... .......... 81% 28.7M 1s\n 48400K .......... .......... .......... .......... .......... 81% 3.45M 1s\n 48450K .......... .......... .......... .......... .......... 81% 9.00M 1s\n 48500K .......... .......... .......... .......... .......... 81% 11.2M 1s\n 48550K .......... .......... .......... .......... .......... 81% 14.2M 1s\n 48600K .......... .......... .......... .......... .......... 81% 23.6M 1s\n 48650K .......... .......... .......... .......... .......... 81% 32.6M 1s\n 48700K .......... .......... .......... .......... .......... 81% 19.4M 1s\n 48750K .......... .......... .......... .......... .......... 81% 43.3M 1s\n 48800K .......... .......... .......... .......... .......... 82% 3.11M 1s\n 48850K .......... .......... .......... .......... .......... 82% 9.56M 1s\n 48900K .......... .......... .......... .......... .......... 82% 12.4M 1s\n 48950K .......... .......... .......... .......... .......... 82% 15.5M 1s\n 49000K .......... .......... .......... .......... .......... 82% 26.3M 1s\n 49050K .......... .......... .......... .......... .......... 82% 32.7M 1s\n 49100K .......... .......... .......... .......... .......... 82% 20.8M 1s\n 49150K .......... .......... .......... .......... .......... 82% 45.0M 1s\n 49200K .......... .......... .......... .......... .......... 82% 2.71M 1s\n 49250K .......... .......... .......... .......... .......... 82% 12.0M 1s\n 49300K .......... .......... .......... .......... .......... 82% 14.9M 1s\n 49350K .......... .......... .......... .......... .......... 82% 17.5M 1s\n 49400K .......... .......... .......... .......... .......... 83% 21.1M 1s\n 49450K .......... .......... .......... .......... .......... 83% 72.5M 1s\n 49500K .......... .......... .......... .......... .......... 83% 18.8M 1s\n 49550K .......... .......... .......... .......... .......... 83% 3.50M 1s\n 49600K .......... .......... .......... .......... .......... 83% 8.53M 1s\n 49650K .......... .......... .......... .......... .......... 83% 11.7M 1s\n 49700K .......... .......... .......... .......... .......... 83% 13.7M 1s\n 49750K .......... .......... .......... .......... .......... 83% 25.4M 1s\n 49800K .......... .......... .......... .......... .......... 83% 25.6M 1s\n 49850K .......... .......... .......... .......... .......... 83% 34.6M 1s\n 49900K .......... .......... .......... .......... .......... 83% 21.5M 1s\n 49950K .......... .......... .......... .......... .......... 84% 3.21M 1s\n 50000K .......... .......... .......... .......... .......... 84% 9.43M 1s\n 50050K .......... .......... .......... .......... .......... 84% 12.3M 1s\n 50100K .......... .......... .......... .......... .......... 84% 15.4M 1s\n 50150K .......... .......... .......... .......... .......... 84% 24.3M 1s\n 50200K .......... .......... .......... .......... .......... 84% 30.6M 1s\n 50250K .......... .......... .......... .......... .......... 84% 26.2M 1s\n 50300K .......... .......... .......... .......... .......... 84% 28.5M 1s\n 50350K .......... .......... .......... .......... .......... 84% 2.81M 1s\n 50400K .......... .......... .......... .......... .......... 84% 11.7M 1s\n 50450K .......... .......... .......... .......... .......... 84% 14.2M 1s\n 50500K .......... .......... .......... .......... .......... 84% 17.4M 1s\n 50550K .......... .......... .......... .......... .......... 85% 22.5M 1s\n 50600K .......... .......... .......... .......... .......... 85% 42.5M 1s\n 50650K .......... .......... .......... .......... .......... 85% 26.8M 1s\n 50700K .......... .......... .......... .......... .......... 85% 3.47M 1s\n 50750K .......... .......... .......... .......... .......... 85% 8.16M 1s\n 50800K .......... .......... .......... .......... .......... 85% 12.2M 1s\n 50850K .......... .......... .......... .......... .......... 85% 13.8M 1s\n 50900K .......... .......... .......... .......... .......... 85% 23.3M 1s\n 50950K .......... .......... .......... .......... .......... 85% 23.3M 1s\n 51000K .......... .......... .......... .......... .......... 85% 43.9M 1s\n 51050K .......... .......... .......... .......... .......... 85% 26.8M 1s\n 51100K .......... .......... .......... .......... .......... 85% 3.11M 1s\n 51150K .......... .......... .......... .......... .......... 86% 9.34M 1s\n 51200K .......... .......... .......... .......... .......... 86% 12.0M 1s\n 51250K .......... .......... .......... .......... .......... 86% 16.9M 1s\n 51300K .......... .......... .......... .......... .......... 86% 22.2M 1s\n 51350K .......... .......... .......... .......... .......... 86% 30.3M 1s\n 51400K .......... .......... .......... .......... .......... 86% 29.3M 1s\n 51450K .......... .......... .......... .......... .......... 86% 28.8M 1s\n 51500K .......... .......... .......... .......... .......... 86% 2.80M 1s\n 51550K .......... .......... .......... .......... .......... 86% 11.4M 1s\n 51600K .......... .......... .......... .......... .......... 86% 13.8M 1s\n 51650K .......... .......... .......... .......... .......... 86% 17.8M 1s\n 51700K .......... .......... .......... .......... .......... 86% 21.2M 1s\n 51750K .......... .......... .......... .......... .......... 87% 39.3M 1s\n 51800K .......... .......... .......... .......... .......... 87% 28.0M 1s\n 51850K .......... .......... .......... .......... .......... 87% 34.8M 1s\n 51900K .......... .......... .......... .......... .......... 87% 2.66M 1s\n 51950K .......... .......... .......... .......... .......... 87% 11.9M 1s\n 52000K .......... .......... .......... .......... .......... 87% 14.3M 1s\n 52050K .......... .......... .......... .......... .......... 87% 21.5M 1s\n 52100K .......... .......... .......... .......... .......... 87% 21.0M 1s\n 52150K .......... .......... .......... .......... .......... 87% 42.0M 1s\n 52200K .......... .......... .......... .......... .......... 87% 30.7M 1s\n 52250K .......... .......... .......... .......... .......... 87% 3.62M 1s\n 52300K .......... .......... .......... .......... .......... 87% 6.50M 1s\n 52350K .......... .......... .......... .......... .......... 88% 12.7M 1s\n 52400K .......... .......... .......... .......... .......... 88% 15.9M 1s\n 52450K .......... .......... .......... .......... .......... 88% 22.2M 1s\n 52500K .......... .......... .......... .......... .......... 88% 25.5M 1s\n 52550K .......... .......... .......... .......... .......... 88% 31.0M 1s\n 52600K .......... .......... .......... .......... .......... 88% 34.0M 1s\n 52650K .......... .......... .......... .......... .......... 88% 3.30M 1s\n 52700K .......... .......... .......... .......... .......... 88% 7.22M 1s\n 52750K .......... .......... .......... .......... .......... 88% 12.3M 1s\n 52800K .......... .......... .......... .......... .......... 88% 21.3M 1s\n 52850K .......... .......... .......... .......... .......... 88% 20.0M 1s\n 52900K .......... .......... .......... .......... .......... 88% 33.5M 1s\n 52950K .......... .......... .......... .......... .......... 89% 25.6M 1s\n 53000K .......... .......... .......... .......... .......... 89% 35.8M 1s\n 53050K .......... .......... .......... .......... .......... 89% 2.96M 1s\n 53100K .......... .......... .......... .......... .......... 89% 8.83M 1s\n 53150K .......... .......... .......... .......... .......... 89% 13.7M 1s\n 53200K .......... .......... .......... .......... .......... 89% 22.5M 1s\n 53250K .......... .......... .......... .......... .......... 89% 19.7M 1s\n 53300K .......... .......... .......... .......... .......... 89% 37.0M 1s\n 53350K .......... .......... .......... .......... .......... 89% 30.4M 1s\n 53400K .......... .......... .......... .......... .......... 89% 3.71M 1s\n 53450K .......... .......... .......... .......... .......... 89% 8.29M 1s\n 53500K .......... .......... .......... .......... .......... 89% 8.93M 1s\n 53550K .......... .......... .......... .......... .......... 90% 15.0M 1s\n 53600K .......... .......... .......... .......... .......... 90% 22.2M 1s\n 53650K .......... .......... .......... .......... .......... 90% 26.5M 1s\n 53700K .......... .......... .......... .......... .......... 90% 30.7M 1s\n 53750K .......... .......... .......... .......... .......... 90% 33.6M 1s\n 53800K .......... .......... .......... .......... .......... 90% 3.37M 1s\n 53850K .......... .......... .......... .......... .......... 90% 8.54M 1s\n 53900K .......... .......... .......... .......... .......... 90% 9.21M 1s\n 53950K .......... .......... .......... .......... .......... 90% 20.9M 1s\n 54000K .......... .......... .......... .......... .......... 90% 22.6M 1s\n 54050K .......... .......... .......... .......... .......... 90% 31.8M 0s\n 54100K .......... .......... .......... .......... .......... 90% 26.3M 0s\n 54150K .......... .......... .......... .......... .......... 91% 29.5M 0s\n 54200K .......... .......... .......... .......... .......... 91% 3.05M 0s\n 54250K .......... .......... .......... .......... .......... 91% 10.4M 0s\n 54300K .......... .......... .......... .......... .......... 91% 10.3M 0s\n 54350K .......... .......... .......... .......... .......... 91% 22.3M 0s\n 54400K .......... .......... .......... .......... .......... 91% 19.6M 0s\n 54450K .......... .......... .......... .......... .......... 91% 29.9M 0s\n 54500K .......... .......... .......... .......... .......... 91% 39.3M 0s\n 54550K .......... .......... .......... .......... .......... 91% 3.67M 0s\n 54600K .......... .......... .......... .......... .......... 91% 8.34M 0s\n 54650K .......... .......... .......... .......... .......... 91% 11.8M 0s\n 54700K .......... .......... .......... .......... .......... 91% 10.7M 0s\n 54750K .......... .......... .......... .......... .......... 92% 22.5M 0s\n 54800K .......... .......... .......... .......... .......... 92% 25.7M 0s\n 54850K .......... .......... .......... .......... .......... 92% 30.3M 0s\n 54900K .......... .......... .......... .......... .......... 92% 35.0M 0s\n 54950K .......... .......... .......... .......... .......... 92% 3.42M 0s\n 55000K .......... .......... .......... .......... .......... 92% 8.47M 0s\n 55050K .......... .......... .......... .......... .......... 92% 12.6M 0s\n 55100K .......... .......... .......... .......... .......... 92% 12.5M 0s\n 55150K .......... .......... .......... .......... .......... 92% 20.8M 0s\n 55200K .......... .......... .......... .......... .......... 92% 36.2M 0s\n 55250K .......... .......... .......... .......... .......... 92% 23.4M 0s\n 55300K .......... .......... .......... .......... .......... 92% 34.5M 0s\n 55350K .......... .......... .......... .......... .......... 93% 3.06M 0s\n 55400K .......... .......... .......... .......... .......... 93% 10.3M 0s\n 55450K .......... .......... .......... .......... .......... 93% 12.4M 0s\n 55500K .......... .......... .......... .......... .......... 93% 15.4M 0s\n 55550K .......... .......... .......... .......... .......... 93% 20.5M 0s\n 55600K .......... .......... .......... .......... .......... 93% 32.2M 0s\n 55650K .......... .......... .......... .......... .......... 93% 31.4M 0s\n 55700K .......... .......... .......... .......... .......... 93% 3.80M 0s\n 55750K .......... .......... .......... .......... .......... 93% 8.10M 0s\n 55800K .......... .......... .......... .......... .......... 93% 11.8M 0s\n 55850K .......... .......... .......... .......... .......... 93% 14.4M 0s\n 55900K .......... .......... .......... .......... .......... 94% 14.3M 0s\n 55950K .......... .......... .......... .......... .......... 94% 25.0M 0s\n 56000K .......... .......... .......... .......... .......... 94% 30.5M 0s\n 56050K .......... .......... .......... .......... .......... 94% 39.2M 0s\n 56100K .......... .......... .......... .......... .......... 94% 3.44M 0s\n 56150K .......... .......... .......... .......... .......... 94% 8.29M 0s\n 56200K .......... .......... .......... .......... .......... 94% 13.0M 0s\n 56250K .......... .......... .......... .......... .......... 94% 14.7M 0s\n 56300K .......... .......... .......... .......... .......... 94% 16.3M 0s\n 56350K .......... .......... .......... .......... .......... 94% 29.9M 0s\n 56400K .......... .......... .......... .......... .......... 94% 28.7M 0s\n 56450K .......... .......... .......... .......... .......... 94% 35.6M 0s\n 56500K .......... .......... .......... .......... .......... 95% 3.06M 0s\n 56550K .......... .......... .......... .......... .......... 95% 10.1M 0s\n 56600K .......... .......... .......... .......... .......... 95% 12.6M 0s\n 56650K .......... .......... .......... .......... .......... 95% 18.6M 0s\n 56700K .......... .......... .......... .......... .......... 95% 15.7M 0s\n 56750K .......... .......... .......... .......... .......... 95% 32.3M 0s\n 56800K .......... .......... .......... .......... .......... 95% 29.4M 0s\n 56850K .......... .......... .......... .......... .......... 95% 3.93M 0s\n 56900K .......... .......... .......... .......... .......... 95% 7.87M 0s\n 56950K .......... .......... .......... .......... .......... 95% 11.5M 0s\n 57000K .......... .......... .......... .......... .......... 95% 14.2M 0s\n 57050K .......... .......... .......... .......... .......... 95% 19.5M 0s\n 57100K .......... .......... .......... .......... .......... 96% 16.2M 0s\n 57150K .......... .......... .......... .......... .......... 96% 35.8M 0s\n 57200K .......... .......... .......... .......... .......... 96% 32.8M 0s\n 57250K .......... .......... .......... .......... .......... 96% 3.52M 0s\n 57300K .......... .......... .......... .......... .......... 96% 8.04M 0s\n 57350K .......... .......... .......... .......... .......... 96% 13.8M 0s\n 57400K .......... .......... .......... .......... .......... 96% 13.6M 0s\n 57450K .......... .......... .......... .......... .......... 96% 21.3M 0s\n 57500K .......... .......... .......... .......... .......... 96% 21.2M 0s\n 57550K .......... .......... .......... .......... .......... 96% 31.4M 0s\n 57600K .......... .......... .......... .......... .......... 96% 29.9M 0s\n 57650K .......... .......... .......... .......... .......... 96% 3.15M 0s\n 57700K .......... .......... .......... .......... .......... 97% 9.63M 0s\n 57750K .......... .......... .......... .......... .......... 97% 13.2M 0s\n 57800K .......... .......... .......... .......... .......... 97% 16.8M 0s\n 57850K .......... .......... .......... .......... .......... 97% 20.0M 0s\n 57900K .......... .......... .......... .......... .......... 97% 23.0M 0s\n 57950K .......... .......... .......... .......... .......... 97% 23.8M 0s\n 58000K .......... .......... .......... .......... .......... 97% 4.12M 0s\n 58050K .......... .......... .......... .......... .......... 97% 7.73M 0s\n 58100K .......... .......... .......... .......... .......... 97% 11.1M 0s\n 58150K .......... .......... .......... .......... .......... 97% 14.5M 0s\n 58200K .......... .......... .......... .......... .......... 97% 19.4M 0s\n 58250K .......... .......... .......... .......... .......... 97% 20.6M 0s\n 58300K .......... .......... .......... .......... .......... 98% 23.2M 0s\n 58350K .......... .......... .......... .......... .......... 98% 31.4M 0s\n 58400K .......... .......... .......... .......... .......... 98% 3.51M 0s\n 58450K .......... .......... .......... .......... .......... 98% 8.61M 0s\n 58500K .......... .......... .......... .......... .......... 98% 13.3M 0s\n 58550K .......... .......... .......... .......... .......... 98% 13.6M 0s\n 58600K .......... .......... .......... .......... .......... 98% 22.4M 0s\n 58650K .......... .......... .......... .......... .......... 98% 20.2M 0s\n 58700K .......... .......... .......... .......... .......... 98% 27.3M 0s\n 58750K .......... .......... .......... .......... .......... 98% 27.9M 0s\n 58800K .......... .......... .......... .......... .......... 98% 3.16M 0s\n 58850K .......... .......... .......... .......... .......... 98% 9.62M 0s\n 58900K .......... .......... .......... .......... .......... 99% 12.8M 0s\n 58950K .......... .......... .......... .......... .......... 99% 17.3M 0s\n 59000K .......... .......... .......... .......... .......... 99% 19.8M 0s\n 59050K .......... .......... .......... .......... .......... 99% 32.2M 0s\n 59100K .......... .......... .......... .......... .......... 99% 20.3M 0s\n 59150K .......... .......... .......... .......... .......... 99% 4.16M 0s\n 59200K .......... .......... .......... .......... .......... 99% 7.88M 0s\n 59250K .......... .......... .......... .......... .......... 99% 11.1M 0s\n 59300K .......... .......... .......... .......... .......... 99% 13.1M 0s\n 59350K .......... .......... .......... .......... .......... 99% 18.7M 0s\n 59400K .......... .......... .......... .......... .......... 99% 21.6M 0s\n 59450K .......... .......... .......... .......... .......... 99% 36.3M 0s\n 59500K .......... .........                                  100% 25.8M=5.4s\n\n2016-05-19 00:46:50 (10.7 MB/s) - &apos;all.tsv&apos; saved [60947802/60947802]\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Cancelled","error":null,"startTime":1.463618755804E12,"submitTime":1.463618601144E12,"finishTime":1.463618810297E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"08f6370f-8b4a-4a2c-98e2-df16affb5052"},{"version":"CommandV1","origId":118507,"guid":"40b0d5b9-c1de-4655-9d74-f33cae9802a0","subtype":"command","commandType":"auto","position":14.6953125,"command":"%sh\npwd","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">/databricks/driver\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.463618845695E12,"submitTime":1.463618690997E12,"finishTime":1.463618845743E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b7e2098f-9f3a-44e6-8572-29e1a78c7c6a"},{"version":"CommandV1","origId":116435,"guid":"dbc6263f-c221-4b97-8475-373e6c0d4e72","subtype":"command","commandType":"auto","position":14.703125,"command":"dbutils.fs.mkdirs(\"dbfs:/datasets/magellan\") //need not be done again!","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res12: Boolean = true\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Cancelled","error":null,"startTime":1.463618810314E12,"submitTime":1.463618638236E12,"finishTime":1.463618810634E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cb4885f9-fa29-4fe9-8971-009ce766d9fc"},{"version":"CommandV1","origId":116436,"guid":"66be2f34-2c5c-493f-84fd-7ec63efda65d","subtype":"command","commandType":"auto","position":14.71875,"command":"dbutils.fs.cp(\"file:/databricks/driver/all.tsv\", \"dbfs:/datasets/magellan/\") ","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res14: Boolean = true\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"java.io.FileNotFoundException: File file:/home/ubuntu/databricks/driver/all.tsv does not exist.","error":"<div class=\"ansiout\">\tat org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:402)\n\tat org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:255)\n\tat com.databricks.backend.daemon.dbutils.FSUtils$.cp(DBUtilsCore.scala:81)\n\tat com.databricks.dbutils_v1.package$fs$.cp(dbutils_v1.scala:86)</div>","startTime":1.46361886648E12,"submitTime":1.463618711815E12,"finishTime":1.463618871863E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"749fa6f5-8b6c-42c3-85a4-9365b8fe4a2c"},{"version":"CommandV1","origId":116437,"guid":"09c80743-9288-4764-a6a5-daa86cdc4890","subtype":"command","commandType":"auto","position":14.734375,"command":"%md \n## Getting SF Neighborhood Data","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451997215E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"fb892ac1-069b-464c-a247-d7d1b397b20e"},{"version":"CommandV1","origId":116438,"guid":"63cd35ca-b917-4a19-8fca-353b649023c8","subtype":"command","commandType":"auto","position":14.7421875,"command":"%sh\nwget http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/2016/datasets/magellan/UberSF/planning_neighborhoods.zip","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">--2016-05-19 00:47:54--  http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/2016/datasets/magellan/UberSF/planning_neighborhoods.zip\nResolving www.math.canterbury.ac.nz (www.math.canterbury.ac.nz)... 132.181.26.35\nConnecting to www.math.canterbury.ac.nz (www.math.canterbury.ac.nz)|132.181.26.35|:80... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 163771 (160K) [application/zip]\nSaving to: &apos;planning_neighborhoods.zip.1&apos;\n\n     0K .......... .......... .......... .......... .......... 31%  449K 0s\n    50K .......... .......... .......... .......... .......... 62%  667K 0s\n   100K .......... .......... .......... .......... .......... 93% 48.4M 0s\n   150K .........                                             100%  271K=0.2s\n\n2016-05-19 00:47:54 (715 KB/s) - &apos;planning_neighborhoods.zip.1&apos; saved [163771/163771]\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463618874502E12,"submitTime":1.463618719839E12,"finishTime":1.463618874929E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"4faf8edc-1f61-4ea0-ba17-98649e29164f"},{"version":"CommandV1","origId":116439,"guid":"c6cc5bab-f346-4d8f-aaba-527077505d0c","subtype":"command","commandType":"auto","position":14.74609375,"command":"%sh\nunzip planning_neighborhoods.zip","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">Archive:  planning_neighborhoods.zip\n  inflating: planning_neighborhoods.dbf  \n  inflating: planning_neighborhoods.shx  \n  inflating: planning_neighborhoods.shp.xml  \n  inflating: planning_neighborhoods.shp  \n  inflating: planning_neighborhoods.sbx  \n  inflating: planning_neighborhoods.sbn  \n  inflating: planning_neighborhoods.prj  \n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463618878503E12,"submitTime":1.463618723812E12,"finishTime":1.46361887853E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"94f50d06-aa5f-4736-8c34-397c71e6d80f"},{"version":"CommandV1","origId":116440,"guid":"4a773235-7b70-44d0-ba8a-acc7a04afbaa","subtype":"command","commandType":"auto","position":14.7470703125,"command":"%sh \nmv planning_neighborhoods.zip orig_planning_neighborhoods.zip","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\"></div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463618884545E12,"submitTime":1.463618729879E12,"finishTime":1.463618884574E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f1951a4f-5cce-4884-a0a5-ee1adb77a629"},{"version":"CommandV1","origId":116441,"guid":"8c5fc05d-1fe6-4410-a9a5-e7455b5df388","subtype":"command","commandType":"auto","position":14.748046875,"command":"%sh\nmkdir SFNbhd && mv planning_nei* SFNbhd && ls \nls SFNbhd","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">SFNbhd\nall.tsv\neventlogs\nlogs\norig_planning_neighborhoods.zip\nplanning_neighborhoods.dbf\nplanning_neighborhoods.prj\nplanning_neighborhoods.sbn\nplanning_neighborhoods.sbx\nplanning_neighborhoods.shp\nplanning_neighborhoods.shp.xml\nplanning_neighborhoods.shx\nplanning_neighborhoods.zip.1\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463618896162E12,"submitTime":1.463618741461E12,"finishTime":1.463618896207E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b0210f3b-11fe-45a9-9b19-7892d876445e"},{"version":"CommandV1","origId":116442,"guid":"b9b31e8c-b676-43a7-9dd2-df0a9d1b1ed0","subtype":"command","commandType":"auto","position":14.74853515625,"command":"dbutils.fs.mkdirs(\"dbfs:/datasets/magellan/SFNbhd\") //need not be done again!","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res15: Boolean = true\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.463618904224E12,"submitTime":1.46361874955E12,"finishTime":1.463618904563E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9349645b-b531-4b52-9f26-0ad7c6e9e37b"},{"version":"CommandV1","origId":116443,"guid":"a56edbfc-fc59-4390-bc04-7a5d7002f76e","subtype":"command","commandType":"auto","position":14.7490234375,"command":"dbutils.fs.cp(\"file:/databricks/driver/SFNbhd/planning_neighborhoods.dbf\", \"dbfs:/datasets/magellan/SFNbhd/\")\ndbutils.fs.cp(\"file:/databricks/driver/SFNbhd/planning_neighborhoods.prj\", \"dbfs:/datasets/magellan/SFNbhd/\")\ndbutils.fs.cp(\"file:/databricks/driver/SFNbhd/planning_neighborhoods.sbn\", \"dbfs:/datasets/magellan/SFNbhd/\")\ndbutils.fs.cp(\"file:/databricks/driver/SFNbhd/planning_neighborhoods.sbx\", \"dbfs:/datasets/magellan/SFNbhd/\")\ndbutils.fs.cp(\"file:/databricks/driver/SFNbhd/planning_neighborhoods.shp\", \"dbfs:/datasets/magellan/SFNbhd/\")\ndbutils.fs.cp(\"file:/databricks/driver/SFNbhd/planning_neighborhoods.shp.xml\", \"dbfs:/datasets/magellan/SFNbhd/\")\ndbutils.fs.cp(\"file:/databricks/driver/SFNbhd/planning_neighborhoods.shx\", \"dbfs:/datasets/magellan/SFNbhd/\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res17: Boolean = true\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"java.io.FileNotFoundException: File file:/home/ubuntu/databricks/driver/SFNbhd/planning_neighborhoods.dbf does not exist.","error":"<div class=\"ansiout\">\tat org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:402)\n\tat org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:255)\n\tat com.databricks.backend.daemon.dbutils.FSUtils$.cp(DBUtilsCore.scala:81)\n\tat com.databricks.dbutils_v1.package$fs$.cp(dbutils_v1.scala:86)</div>","startTime":1.463618949733E12,"submitTime":1.463618795067E12,"finishTime":1.463618953952E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c25c3144-426c-4218-9f4a-d6aeec03efa5"},{"version":"CommandV1","origId":116444,"guid":"b19dc50f-e4a5-44ac-827f-cd468942b2c3","subtype":"command","commandType":"auto","position":14.74951171875,"command":"display(dbutils.fs.ls(\"dbfs:/datasets/magellan/SFNbhd/\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.dbf","planning_neighborhoods.dbf",1028.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.prj","planning_neighborhoods.prj",567.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.sbn","planning_neighborhoods.sbn",516.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.sbx","planning_neighborhoods.sbx",164.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.shp","planning_neighborhoods.shp",214576.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.shp.xml","planning_neighborhoods.shp.xml",21958.0],["dbfs:/datasets/magellan/SFNbhd/planning_neighborhoods.shx","planning_neighborhoods.shx",396.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.463618957613E12,"submitTime":1.463618802946E12,"finishTime":1.463618963351E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"1a5022ce-0cd3-40d7-abe0-95fcec2f3407"},{"version":"CommandV1","origId":116445,"guid":"16156c29-2925-43a6-abe5-8bd35aeb3a1e","subtype":"command","commandType":"auto","position":19.0,"command":"%md \n### End of downloading and putting data in dbfs","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.454451997648E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":null,"showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b77726cf-0c52-4baa-93d2-0e7c560d75cd"},{"version":"CommandV1","origId":116446,"guid":"bb705818-6a91-4d0d-a725-9757a2fa0b33","subtype":"command","commandType":"auto","position":20.0,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n### prepared by Dillon George, [Raazesh Sainudiin](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [Sivanand Sivaram](https://www.linkedin.com/in/sivanand)\n\n*supported by* [![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/databricks_logoTM_200px.png)](https://databricks.com/)\nand \n[![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/AWS_logoTM_200px.png)](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b375f510-4e84-482b-a83c-6a6c9d727756"}],"dashboards":[],"guid":"35d95e97-771a-4182-a346-31cfee69e52b","globalVars":{},"iPythonMetadata":null,"inputWidgets":{}};</script>
<script
 src="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js"
 onerror="window.mainJsLoadError = true;"></script>
</head>
<body>
  <script>
if (window.mainJsLoadError) {
  var u = 'https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js';
  var b = document.getElementsByTagName('body')[0];
  var c = document.createElement('div');
  c.innerHTML = ('<h1>Network Error</h1>' +
    '<p><b>Please check your network connection and try again.</b></p>' +
    '<p>Could not load a required resource: ' + u + '</p>');
  c.style.margin = '30px';
  c.style.padding = '20px 50px';
  c.style.backgroundColor = '#f5f5f5';
  c.style.borderRadius = '5px';
  b.appendChild(c);
}
</script>
</body>
</html>