<!DOCTYPE html> <html> <head> <meta name="databricks-html-version" content="1"> <title>026_GraphFramesUserGuide - Databricks</title> <meta charset="utf-8"> <meta name="google" content="notranslate"> <meta http-equiv="Content-Language" content="en"> <meta http-equiv="Content-Type" content="text/html; charset=UTF8"> <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Code+Pro:400,700"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/css/bootstrap.min.css"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/jquery-ui-bundle/jquery-ui.min.css"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/main.css"> <link rel="stylesheet" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/print.css" media="print"> <link rel="icon" type="image/png" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/img/favicon.ico"/> <script>window.settings = {"sparkDocsSearchGoogleCx":"004588677886978090460:_rj0wilqwdm","dbcForumURL":"http://forums.databricks.com/","dbfsS3Host":"https://databricks-prod-storage-sydney.s3.amazonaws.com","enableThirdPartyApplicationsUI":false,"enableClusterAcls":false,"notebookRevisionVisibilityHorizon":0,"enableTableHandler":true,"isAdmin":true,"enableLargeResultDownload":false,"nameAndEmail":"Raazesh Sainudiin (r.sainudiin@math.canterbury.ac.nz)","enablePresentationTimerConfig":true,"enableFullTextSearch":true,"enableElasticSparkUI":true,"clusters":true,"hideOffHeapCache":false,"applications":false,"useStaticGuide":false,"fileStoreBase":"FileStore","configurableSparkOptionsSpec":[{"keyPattern":"spark\\.kryo(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.kryo.*","valuePatternDisplay":"*","description":"Configuration options for Kryo serialization"},{"keyPattern":"spark\\.io\\.compression\\.codec","valuePattern":"(lzf|snappy|org\\.apache\\.spark\\.io\\.LZFCompressionCodec|org\\.apache\\.spark\\.io\\.SnappyCompressionCodec)","keyPatternDisplay":"spark.io.compression.codec","valuePatternDisplay":"snappy|lzf","description":"The codec used to compress internal data such as RDD partitions, broadcast variables and shuffle outputs."},{"keyPattern":"spark\\.serializer","valuePattern":"(org\\.apache\\.spark\\.serializer\\.JavaSerializer|org\\.apache\\.spark\\.serializer\\.KryoSerializer)","keyPatternDisplay":"spark.serializer","valuePatternDisplay":"org.apache.spark.serializer.JavaSerializer|org.apache.spark.serializer.KryoSerializer","description":"Class to use for serializing objects that will be sent over the network or need to be cached in serialized form."},{"keyPattern":"spark\\.rdd\\.compress","valuePattern":"(true|false)","keyPatternDisplay":"spark.rdd.compress","valuePatternDisplay":"true|false","description":"Whether to compress serialized RDD partitions (e.g. for StorageLevel.MEMORY_ONLY_SER). Can save substantial space at the cost of some extra CPU time."},{"keyPattern":"spark\\.speculation","valuePattern":"(true|false)","keyPatternDisplay":"spark.speculation","valuePatternDisplay":"true|false","description":"Whether to use speculation (recommended off for streaming)"},{"keyPattern":"spark\\.es(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"es(\\.([^\\.]+))+","valuePattern":".*","keyPatternDisplay":"es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"spark\\.(storage|shuffle)\\.memoryFraction","valuePattern":"0?\\.0*([1-9])([0-9])*","keyPatternDisplay":"spark.(storage|shuffle).memoryFraction","valuePatternDisplay":"(0.0,1.0)","description":"Fraction of Java heap to use for Spark's shuffle or storage"},{"keyPattern":"spark\\.streaming\\.backpressure\\.enabled","valuePattern":"(true|false)","keyPatternDisplay":"spark.streaming.backpressure.enabled","valuePatternDisplay":"true|false","description":"Enables or disables Spark Streaming's internal backpressure mechanism (since 1.5). This enables the Spark Streaming to control the receiving rate based on the current batch scheduling delays and processing times so that the system receives only as fast as the system can process. Internally, this dynamically sets the maximum receiving rate of receivers. This rate is upper bounded by the values `spark.streaming.receiver.maxRate` and `spark.streaming.kafka.maxRatePerPartition` if they are set."},{"keyPattern":"spark\\.streaming\\.receiver\\.maxRate","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.receiver.maxRate","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which each receiver will receive data. Effectively, each stream will consume at most this number of records per second. Setting this configuration to 0 or a negative number will put no limit on the rate. See the deployment guide in the Spark Streaming programing guide for mode details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRatePerPartition","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRatePerPartition","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which data will be read from each Kafka partition when using the Kafka direct stream API introduced in Spark 1.3. See the Kafka Integration guide for more details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRetries","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRetries","valuePatternDisplay":"numeric","description":"Maximum number of consecutive retries the driver will make in order to find the latest offsets on the leader of each partition (a default value of 1 means that the driver will make a maximum of 2 attempts). Only applies to the Kafka direct stream API introduced in Spark 1.3."},{"keyPattern":"spark\\.streaming\\.ui\\.retainedBatches","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.ui.retainedBatches","valuePatternDisplay":"numeric","description":"How many batches the Spark Streaming UI and status APIs remember before garbage collecting."}],"enableReactNotebookComments":true,"enableResetPassword":true,"enableJobsSparkUpgrade":true,"sparkVersions":[{"key":"1.3.x-ubuntu15.10","displayName":"Spark 1.3.0","packageLabel":"spark-1.3-jenkins-ip-10-30-9-162-U0c2673ac85-Sa2ee4664b2-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.4.x-ubuntu15.10","displayName":"Spark 1.4.1","packageLabel":"spark-1.4-jenkins-ip-10-30-9-162-U0c2673ac85-S33a1e4b9c6-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.6.x-ubuntu15.10","displayName":"Spark 1.6.0","packageLabel":"spark-1.6-jenkins-ip-10-30-9-162-U0c2673ac85-Scabba801f3-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"master","displayName":"Spark master (dev)","packageLabel":"","upgradable":true,"deprecated":false,"customerVisible":false}],"enableRestrictedClusterCreation":false,"enableFeedback":false,"defaultNumWorkers":8,"serverContinuationTimeoutMillis":10000,"driverStderrFilePrefix":"stderr","driverStdoutFilePrefix":"stdout","enableSparkDocsSearch":true,"prefetchSidebarNodes":true,"sparkHistoryServerEnabled":true,"sanitizeMarkdownHtml":true,"enableIPythonImportExport":true,"enableNotebookHistoryDiffing":true,"branch":"2.12.3","accountsLimit":-1,"enableNotebookGitBranching":true,"local":false,"displayDefaultContainerMemoryGB":6,"deploymentMode":"production","useSpotForWorkers":false,"enableUserInviteWorkflow":false,"enableStaticNotebooks":true,"dbcGuideURL":"#workspace/databricks_guide/00 Welcome to Databricks","enableCssTransitions":true,"pricingURL":"https://databricks.com/product/pricing","enableClusterAclsConfig":false,"orgId":0,"enableNotebookGitVersioning":true,"files":"files/","enableDriverLogsUI":true,"disableLegacyDashboards":false,"enableWorkspaceAclsConfig":true,"dropzoneMaxFileSize":4096,"enableNewDashboardViews":false,"driverLog4jFilePrefix":"log4j","enableMavenLibraries":true,"displayRowLimit":1000,"defaultSparkVersion":{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},"clusterPublisherRootId":5,"enableLatestJobRunResultPermalink":true,"disallowAddingAdmins":false,"enableSparkConfUI":true,"enableOrgSwitcherUI":false,"clustersLimit":-1,"enableJdbcImport":true,"logfiles":"logfiles/","enableWebappSharding":false,"enableClusterDeltaUpdates":true,"csrfToken":"4c37e4ff-a908-4b05-8c61-2b76819fa34c","useFixedStaticNotebookVersionForDevelopment":false,"enableBasicReactDialogBoxes":true,"requireEmailUserName":true,"enableDashboardViews":false,"dbcFeedbackURL":"http://feedback.databricks.com/forums/263785-product-feedback","enableWorkspaceAclService":true,"someName":"Raazesh Sainudiin","enableWorkspaceAcls":true,"gitHash":"0c2673ac858e227cad536fdb45d140aeded238db","userFullname":"Raazesh Sainudiin","enableClusterCreatePage":false,"enableImportFromUrl":true,"enableMiniClusters":false,"enableWebSocketDeltaUpdates":true,"enableDebugUI":false,"showHiddenSparkVersions":false,"allowNonAdminUsers":true,"userId":100005,"dbcSupportURL":"","staticNotebookResourceUrl":"https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/","enableSparkPackages":true,"enableHybridClusterType":false,"enableNotebookHistoryUI":true,"availableWorkspaces":[{"name":"Workspace 0","orgId":0}],"enableFolderHtmlExport":true,"enableSparkVersionsUI":true,"databricksGuideStaticUrl":"","enableHybridClusters":true,"notebookLoadingBackground":"#fff","enableNewJobRunDetailsPage":true,"enableDashboardExport":true,"user":"r.sainudiin@math.canterbury.ac.nz","enableServerAutoComplete":true,"enableStaticHtmlImport":true,"defaultMemoryPerContainerMB":6000,"enablePresenceUI":true,"tablesPublisherRootId":7,"enableNewInputWidgetUI":false,"accounts":true,"enableNewProgressReportUI":true,"defaultCoresPerContainer":4};</script> <script>var __DATABRICKS_NOTEBOOK_MODEL = {"version":"NotebookV1","origId":104110,"name":"026_GraphFramesUserGuide","language":"scala","commands":[{"version":"CommandV1","origId":106401,"guid":"492b93a3-9270-4a9f-9cb2-974f15c4421e","subtype":"command","commandType":"auto","position":0.25,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n### prepared by [Raazesh Sainudiin](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [Sivanand Sivaram](https://www.linkedin.com/in/sivanand)\n\n*supported by* [](https://databricks.com/)\nand \n[](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"220463af-a147-454c-bafc-ee8a380bc2a7"},{"version":"CommandV1","origId":129737,"guid":"524d3e76-4d1f-4da5-80a3-993ab9d568cf","subtype":"command","commandType":"auto","position":0.375,"command":"%md\nThe [html source url](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/db/week8/15_GraphX/026_GraphFramesUserGuide.html) of this databricks notebook and its recorded Uji :\n\n[](https://www.youtube.com/v/RbgXUf6KCxY?rel=0&autoplay=1&modestbranding=1&start=0&end=4568)\n","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b9969745-2f05-46cc-960a-b6c3776c58f1"},{"version":"CommandV1","origId":104167,"guid":"9dbfafaf-ae8d-48e7-8e50-5d6f14ada09c","subtype":"command","commandType":"auto","position":0.5,"command":"%md\nThis is an augmentation of [http://go.databricks.com/hubfs/notebooks/3-GraphFrames-User-Guide-scala.html](http://go.databricks.com/hubfs/notebooks/3-GraphFrames-User-Guide-scala.html)\n\nSee:\n* [https://amplab.cs.berkeley.edu/wp-content/uploads/2014/09/graphx.pdf](https://amplab.cs.berkeley.edu/wp-content/uploads/2014/09/graphx.pdf)\n* [https://amplab.github.io/graphx/](https://amplab.github.io/graphx/)\n* [https://spark.apache.org/docs/latest/graphx-programming-guide.html](https://spark.apache.org/docs/latest/graphx-programming-guide.html)\n* [https://databricks.com/blog/2016/03/03/introducing-graphframes.html](https://databricks.com/blog/2016/03/03/introducing-graphframes.html)\n* [https://databricks.com/blog/2016/03/16/on-time-flight-performance-with-spark-graphframes.html](https://databricks.com/blog/2016/03/16/on-time-flight-performance-with-spark-graphframes.html)\n* [http://ampcamp.berkeley.edu/big-data-mini-course/graph-analytics-with-graphx.html](http://ampcamp.berkeley.edu/big-data-mini-course/graph-analytics-with-graphx.html)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f281f9ba-6aa9-4a97-b228-f1473f61d43e"},{"version":"CommandV1","origId":106400,"guid":"1c16f8ae-7cb3-4a69-a72a-cefc252cba10","subtype":"command","commandType":"auto","position":0.5546875,"command":"//This allows easy embedding of publicly available information into any other notebook\n//when viewing in git-book just ignore this block - you may have to manually chase the URL in frameIt(\"URL\").\n//Example usage:\n// displayHTML(frameIt(\"https://en.wikipedia.org/wiki/Latent_Dirichlet_allocation#Topics_in_LDA\",250))\ndef frameIt( u:String, h:Int ) : String = {\n \"\"\"<iframe \n src=\"\"\"\"+ u+\"\"\"\"\n width=\"95%\" height=\"\"\"\" + h + \"\"\"\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/index.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>\"\"\"\n }\ndisplayHTML(frameIt(\"https://amplab.github.io/graphx/\",700))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"https://amplab.github.io/graphx/\"\n width=\"95%\" height=\"700\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.4624201095E12,"submitTime":1.462419939499E12,"finishTime":1.462420109611E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"aab09255-44e2-4b03-954f-dd4f4069c5bb"},{"version":"CommandV1","origId":106399,"guid":"4dcfd442-7d1a-407b-9807-8ef21085aee9","subtype":"command","commandType":"auto","position":0.609375,"command":"displayHTML(frameIt(\"https://spark.apache.org/docs/latest/graphx-programming-guide.html#optimized-representation\",800))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"https://spark.apache.org/docs/latest/graphx-programming-guide.html#optimized-representation\"\n width=\"95%\" height=\"800\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462419798364E12,"submitTime":1.462419628368E12,"finishTime":1.462419798482E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"45af0e04-058f-47c9-835f-1c6ec3dfe01b"},{"version":"CommandV1","origId":104112,"guid":"8f7146bb-28e6-439a-afed-e23192143032","subtype":"command","commandType":"auto","position":0.71875,"command":"%md \n#GraphFrames User Guide (Scala)\n\nGraphFrames is a package for Apache Spark which provides DataFrame-based Graphs. It provides high-level APIs in Scala, Java, and Python. It aims to provide both the functionality of GraphX and extended functionality taking advantage of Spark DataFrames. This extended functionality includes motif finding, DataFrame-based serialization, and highly expressive graph queries.\n\nThe GraphFrames package is available from [Spark Packages](http://spark-packages.org/package/graphframes/graphframes).\n\nThis notebook demonstrates examples from the [GraphFrames User Guide](http://graphframes.github.io/user-guide.html).","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375236878E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2d0e136a-8388-4c2c-b3fc-90b427eb90f5"},{"version":"CommandV1","origId":106340,"guid":"7a583d2d-c568-485b-befd-7ca76263e929","subtype":"command","commandType":"auto","position":0.9375,"command":"displayHTML(frameIt(\"https://databricks.com/blog/2016/03/03/introducing-graphframes.html\",500))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"https://databricks.com/blog/2016/03/03/introducing-graphframes.html\"\n width=\"95%\" height=\"500\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462416652806E12,"submitTime":1.462416482947E12,"finishTime":1.462416652944E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"ea5a611d-34fb-474d-9b3b-72aa45638fe6"},{"version":"CommandV1","origId":104113,"guid":"415b20b2-3eae-4925-adb8-12a81dc05bcf","subtype":"command","commandType":"auto","position":1.25,"command":"import org.apache.spark.sql._\nimport org.apache.spark.sql.functions._\n\nimport org.graphframes._","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">import org.apache.spark.sql._\nimport org.apache.spark.sql.functions._\nimport org.graphframes._\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\"><console>:35: error: object graphframes is not a member of package org\n import org.graphframes._\n ^\n<console>:36: error: object graphframes is not a member of package org\n import org.graphframes.examples\n ^\n</div>","error":null,"startTime":1.462423927178E12,"submitTime":1.462423756997E12,"finishTime":1.462423933009E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f2e1e9aa-0bf4-4191-a26a-7171a04e231f"},{"version":"CommandV1","origId":104114,"guid":"9d49ab17-6de7-43de-9511-f8a1100f06be","subtype":"command","commandType":"auto","position":1.5,"command":"%md\n##Creating GraphFrames\n\nLet us try to create an example social network from the blog: \n* [https://databricks.com/blog/2016/03/03/introducing-graphframes.html](https://databricks.com/blog/2016/03/03/introducing-graphframes.html).\n\n\n\nUsers can create GraphFrames from vertex and edge DataFrames.\n\n* **Vertex DataFrame:** A vertex DataFrame should contain a special column named `id` which specifies unique IDs for each vertex in the graph.\n* **Edge DataFrame:** An edge DataFrame should contain two special columns: `src` (source vertex ID of edge) and `dst` (destination vertex ID of edge).\n\nBoth DataFrames can have arbitrary other columns. Those columns can represent vertex and edge attributes.\n\nIn our example, we can use a GraphFrame can store data or properties associated with each vertex and edge. \n\nIn our social network, each user might have an age and name, and each connection might have a relationship type.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375237064E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d1d983ce-5201-4ea1-bf98-fa11b551f47a"},{"version":"CommandV1","origId":104115,"guid":"ecaf3d47-738d-4d50-81b2-086fbe780bc1","subtype":"command","commandType":"auto","position":2.0,"command":"%md\nCreate the vertices and edges","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"335b6387-c91e-4664-a579-3de651965098"},{"version":"CommandV1","origId":104116,"guid":"e14c2be8-d3ea-468f-8548-4dd4af0edae5","subtype":"command","commandType":"auto","position":2.5,"command":"// Vertex DataFrame\nval v = sqlContext.createDataFrame(List(\n (\"a\", \"Alice\", 34),\n (\"b\", \"Bob\", 36),\n (\"c\", \"Charlie\", 30),\n (\"d\", \"David\", 29),\n (\"e\", \"Esther\", 32),\n (\"f\", \"Fanny\", 36),\n (\"g\", \"Gabby\", 60)\n)).toDF(\"id\", \"name\", \"age\")\n// Edge DataFrame\nval e = sqlContext.createDataFrame(List(\n (\"a\", \"b\", \"friend\"),\n (\"b\", \"c\", \"follow\"),\n (\"c\", \"b\", \"follow\"),\n (\"f\", \"c\", \"follow\"),\n (\"e\", \"f\", \"follow\"),\n (\"e\", \"d\", \"friend\"),\n (\"d\", \"a\", \"friend\"),\n (\"a\", \"e\", \"friend\")\n)).toDF(\"src\", \"dst\", \"relationship\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">v: org.apache.spark.sql.DataFrame = [id: string, name: string, age: int]\ne: org.apache.spark.sql.DataFrame = [src: string, dst: string, relationship: string]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.462424136683E12,"submitTime":1.462423966515E12,"finishTime":1.462424137318E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"222d4037-b350-458b-bd2c-1407ee386873"},{"version":"CommandV1","origId":104117,"guid":"4bbba40c-b111-4c09-be79-50e6ea002762","subtype":"command","commandType":"auto","position":3.0,"command":"%md \nLet's create a graph from these vertices and these edges:","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7fa42626-7a0e-474e-9517-cd4981589408"},{"version":"CommandV1","origId":104118,"guid":"0456859b-e5d1-4f99-aed6-f1211f4dfe1a","subtype":"command","commandType":"auto","position":3.5,"command":"val g = GraphFrame(v, e)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">g: org.graphframes.GraphFrame = GraphFrame(v:[id: string, name: string, age: int], e:[src: string, dst: string, relationship: string])\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\"><console>:36: error: not found: value GraphFrame\n val g = GraphFrame(v, e)\n ^\n</div>","error":null,"startTime":1.46242418793E12,"submitTime":1.462424017752E12,"finishTime":1.462424188197E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"1126d068-b892-4f21-aea3-276a8cc31db2"},{"version":"CommandV1","origId":106381,"guid":"1ee5717c-7ada-4688-a1a5-f3ce9c0945a4","subtype":"command","commandType":"auto","position":3.75,"command":"%md\nLet's use the d3.graphs to visualise graphs (recall the D3 graphs in wiki-click example).","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"ee58e3b2-dbda-4293-a825-bf0d4d807a15"},{"version":"CommandV1","origId":106370,"guid":"2f4c65d1-e3f2-4a13-937f-7219ea7fa474","subtype":"command","commandType":"auto","position":4.0,"command":"package d3\n// We use a package object so that we can define top level classes like Edge that need to be used in other cells\n\nimport org.apache.spark.sql._\nimport com.databricks.backend.daemon.driver.EnhancedRDDFunctions.displayHTML\n\ncase class Edge(src: String, dest: String, count: Long)\n\ncase class Node(name: String)\ncase class Link(source: Int, target: Int, value: Long)\ncase class Graph(nodes: Seq[Node], links: Seq[Link])\n\nobject graphs {\nval sqlContext = SQLContext.getOrCreate(org.apache.spark.SparkContext.getOrCreate()) \nimport sqlContext.implicits._\n \ndef force(clicks: Dataset[Edge], height: Int = 100, width: Int = 960): Unit = {\n val data = clicks.collect()\n val nodes = (data.map(_.src) ++ data.map(_.dest)).map(_.replaceAll(\"_\", \" \")).toSet.toSeq.map(Node)\n val links = data.map { t =>\n Link(nodes.indexWhere(_.name == t.src.replaceAll(\"_\", \" \")), nodes.indexWhere(_.name == t.dest.replaceAll(\"_\", \" \")), t.count / 20 + 1)\n }\n showGraph(height, width, Seq(Graph(nodes, links)).toDF().toJSON.first())\n}\n\n/**\n * Displays a force directed graph using d3\n * input: {\"nodes\": [{\"name\": \"...\"}], \"links\": [{\"source\": 1, \"target\": 2, \"value\": 0}]}\n */\ndef showGraph(height: Int, width: Int, graph: String): Unit = {\n\ndisplayHTML(s\"\"\"\n<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n <title>Polish Books Themes - an Interactive Map</title>\n <meta charset=\"utf-8\">\n<style>\n\n.node_circle {\n stroke: #777;\n stroke-width: 1.3px;\n}\n\n.node_label {\n pointer-events: none;\n}\n\n.link {\n stroke: #777;\n stroke-opacity: .2;\n}\n\n.node_count {\n stroke: #777;\n stroke-width: 1.0px;\n fill: #999;\n}\n\ntext.legend {\n font-family: Verdana;\n font-size: 13px;\n fill: #000;\n}\n\n.node text {\n font-family: \"Helvetica Neue\",\"Helvetica\",\"Arial\",sans-serif;\n font-size: 17px;\n font-weight: 200;\n}\n\n</style>\n</head>\n\n<body>\n<script src=\"//d3js.org/d3.v3.min.js\"><\/script>\n<script>\n\nvar graph = $graph;\n\nvar width = $width,\n height = $height;\n\nvar color = d3.scale.category20();\n\nvar force = d3.layout.force()\n .charge(-700)\n .linkDistance(180)\n .size([width, height]);\n\nvar svg = d3.select(\"body\").append(\"svg\")\n .attr(\"width\", width)\n .attr(\"height\", height);\n \nforce\n .nodes(graph.nodes)\n .links(graph.links)\n .start();\n\nvar link = svg.selectAll(\".link\")\n .data(graph.links)\n .enter().append(\"line\")\n .attr(\"class\", \"link\")\n .style(\"stroke-width\", function(d) { return Math.sqrt(d.value); });\n\nvar node = svg.selectAll(\".node\")\n .data(graph.nodes)\n .enter().append(\"g\")\n .attr(\"class\", \"node\")\n .call(force.drag);\n\nnode.append(\"circle\")\n .attr(\"r\", 10)\n .style(\"fill\", function (d) {\n if (d.name.startsWith(\"other\")) { return color(1); } else { return color(2); };\n})\n\nnode.append(\"text\")\n .attr(\"dx\", 10)\n .attr(\"dy\", \".35em\")\n .text(function(d) { return d.name });\n \n//Now we are giving the SVGs co-ordinates - the force layout is generating the co-ordinates which this code is using to update the attributes of the SVG elements\nforce.on(\"tick\", function () {\n link.attr(\"x1\", function (d) {\n return d.source.x;\n })\n .attr(\"y1\", function (d) {\n return d.source.y;\n })\n .attr(\"x2\", function (d) {\n return d.target.x;\n })\n .attr(\"y2\", function (d) {\n return d.target.y;\n });\n d3.selectAll(\"circle\").attr(\"cx\", function (d) {\n return d.x;\n })\n .attr(\"cy\", function (d) {\n return d.y;\n });\n d3.selectAll(\"text\").attr(\"x\", function (d) {\n return d.x;\n })\n .attr(\"y\", function (d) {\n return d.y;\n });\n});\n<\/script>\n</html>\n\"\"\")\n}\n \n def help() = {\ndisplayHTML(\"\"\"\n<p>\nProduces a force-directed graph given a collection of edges of the following form:</br>\n<tt><font color=\"#a71d5d\">case class</font> <font color=\"#795da3\">Edge</font>(<font color=\"#ed6a43\">src</font>: <font color=\"#a71d5d\">String</font>, <font color=\"#ed6a43\">dest</font>: <font color=\"#a71d5d\">String</font>, <font color=\"#ed6a43\">count</font>: <font color=\"#a71d5d\">Long</font>)</tt>\n</p>\n<p>Usage:<br/>\n<tt><font color=\"#a71d5d\">import</font> <font color=\"#ed6a43\">d3._</font></tt><br/>\n<tt><font color=\"#795da3\">graphs.force</font>(</br>\n <font color=\"#ed6a43\">height</font> = <font color=\"#795da3\">500</font>,<br/>\n <font color=\"#ed6a43\">width</font> = <font color=\"#795da3\">500</font>,<br/>\n <font color=\"#ed6a43\">clicks</font>: <font color=\"#795da3\">Dataset</font>[<font color=\"#795da3\">Edge</font>])</tt>\n</p>\"\"\")\n }\n}","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">Warning: classes defined within packages cannot be redefined without a cluster restart.\nCompilation successful.\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462424221313E12,"submitTime":1.462424051113E12,"finishTime":1.462424223773E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"6978a74b-57bb-4f72-8ab9-0705e224db96"},{"version":"CommandV1","origId":106371,"guid":"11a36028-3bf5-4113-a308-49d86ba45e29","subtype":"command","commandType":"auto","position":4.4375,"command":"d3.graphs.help()","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"\n<p>\nProduces a force-directed graph given a collection of edges of the following form:</br>\n<tt><font color=\"#a71d5d\">case class</font> <font color=\"#795da3\">Edge</font>(<font color=\"#ed6a43\">src</font>: <font color=\"#a71d5d\">String</font>, <font color=\"#ed6a43\">dest</font>: <font color=\"#a71d5d\">String</font>, <font color=\"#ed6a43\">count</font>: <font color=\"#a71d5d\">Long</font>)</tt>\n</p>\n<p>Usage:<br/>\n<tt><font color=\"#a71d5d\">import</font> <font color=\"#ed6a43\">d3._</font></tt><br/>\n<tt><font color=\"#795da3\">graphs.force</font>(</br>\n <font color=\"#ed6a43\">height</font> = <font color=\"#795da3\">500</font>,<br/>\n <font color=\"#ed6a43\">width</font> = <font color=\"#795da3\">500</font>,<br/>\n <font color=\"#ed6a43\">clicks</font>: <font color=\"#795da3\">Dataset</font>[<font color=\"#795da3\">Edge</font>])</tt>\n</p>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462424227188E12,"submitTime":1.462424056992E12,"finishTime":1.462424227392E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7aa53f34-86e4-4ff0-8b4b-cb2c48d24787"},{"version":"CommandV1","origId":106373,"guid":"3789b09f-be74-4e65-86e9-3b25f393fc78","subtype":"command","commandType":"auto","position":4.546875,"command":"import org.apache.spark.sql.functions.lit // import the lit function in sql\nval gE= g.edges.select($\"src\", $\"dst\".as(\"dest\"), lit(1L).as(\"count\")) // for us the column count is just an edge incidence","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">import org.apache.spark.sql.functions.lit\ngE: org.apache.spark.sql.DataFrame = [src: string, dest: string, count: bigint]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\">Unclosed block</div>","error":null,"startTime":1.462424303842E12,"submitTime":1.462424133651E12,"finishTime":1.462424304134E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"6daf882f-7467-4f48-8574-2c3ad7b5fc9f"},{"version":"CommandV1","origId":106378,"guid":"3e013542-cad7-46a8-933b-f1ce0f315d85","subtype":"command","commandType":"auto","position":4.6015625,"command":"display(gE)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","b",1.0],["b","c",1.0],["c","b",1.0],["f","c",1.0],["e","f",1.0],["e","d",1.0],["d","a",1.0],["a","e",1.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"src","type":"\"string\""},{"name":"dest","type":"\"string\""},{"name":"count","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":null,"error":null,"startTime":1.46242430973E12,"submitTime":1.462424139532E12,"finishTime":1.462424309917E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d41cd426-e28a-4ace-932f-58a92deff89d"},{"version":"CommandV1","origId":106372,"guid":"402cfee8-582e-4d04-a06e-df20f9560d63","subtype":"command","commandType":"auto","position":4.65625,"command":"d3.graphs.force(\n height = 500,\n width = 500,\n clicks = gE.as[d3.Edge])","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"\n<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n <title>Polish Books Themes - an Interactive Map</title>\n <meta charset=\"utf-8\">\n<style>\n\n.node_circle {\n stroke: #777;\n stroke-width: 1.3px;\n}\n\n.node_label {\n pointer-events: none;\n}\n\n.link {\n stroke: #777;\n stroke-opacity: .2;\n}\n\n.node_count {\n stroke: #777;\n stroke-width: 1.0px;\n fill: #999;\n}\n\ntext.legend {\n font-family: Verdana;\n font-size: 13px;\n fill: #000;\n}\n\n.node text {\n font-family: \"Helvetica Neue\",\"Helvetica\",\"Arial\",sans-serif;\n font-size: 17px;\n font-weight: 200;\n}\n\n</style>\n</head>\n\n<body>\n<script src=\"//d3js.org/d3.v3.min.js\"><\/script>\n<script>\n\nvar graph = {\"nodes\":[{\"name\":\"e\"},{\"name\":\"f\"},{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"},{\"name\":\"d\"}],\"links\":[{\"source\":2,\"target\":3,\"value\":1},{\"source\":3,\"target\":4,\"value\":1},{\"source\":4,\"target\":3,\"value\":1},{\"source\":1,\"target\":4,\"value\":1},{\"source\":0,\"target\":1,\"value\":1},{\"source\":0,\"target\":5,\"value\":1},{\"source\":5,\"target\":2,\"value\":1},{\"source\":2,\"target\":0,\"value\":1}]};\n\nvar width = 500,\n height = 500;\n\nvar color = d3.scale.category20();\n\nvar force = d3.layout.force()\n .charge(-700)\n .linkDistance(180)\n .size([width, height]);\n\nvar svg = d3.select(\"body\").append(\"svg\")\n .attr(\"width\", width)\n .attr(\"height\", height);\n \nforce\n .nodes(graph.nodes)\n .links(graph.links)\n .start();\n\nvar link = svg.selectAll(\".link\")\n .data(graph.links)\n .enter().append(\"line\")\n .attr(\"class\", \"link\")\n .style(\"stroke-width\", function(d) { return Math.sqrt(d.value); });\n\nvar node = svg.selectAll(\".node\")\n .data(graph.nodes)\n .enter().append(\"g\")\n .attr(\"class\", \"node\")\n .call(force.drag);\n\nnode.append(\"circle\")\n .attr(\"r\", 10)\n .style(\"fill\", function (d) {\n if (d.name.startsWith(\"other\")) { return color(1); } else { return color(2); };\n})\n\nnode.append(\"text\")\n .attr(\"dx\", 10)\n .attr(\"dy\", \".35em\")\n .text(function(d) { return d.name });\n \n//Now we are giving the SVGs co-ordinates - the force layout is generating the co-ordinates which this code is using to update the attributes of the SVG elements\nforce.on(\"tick\", function () {\n link.attr(\"x1\", function (d) {\n return d.source.x;\n })\n .attr(\"y1\", function (d) {\n return d.source.y;\n })\n .attr(\"x2\", function (d) {\n return d.target.x;\n })\n .attr(\"y2\", function (d) {\n return d.target.y;\n });\n d3.selectAll(\"circle\").attr(\"cx\", function (d) {\n return d.x;\n })\n .attr(\"cy\", function (d) {\n return d.y;\n });\n d3.selectAll(\"text\").attr(\"x\", function (d) {\n return d.x;\n })\n .attr(\"y\", function (d) {\n return d.y;\n });\n});\n<\/script>\n</html>\n","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"java.lang.RuntimeException: Error while decoding: java.lang.UnsupportedOperationException: Cannot evaluate expression: upcast('count,LongType,- field (class: \"scala.Long\", name: \"count\"),- root class: \"d3.Edge\")","error":"<div class=\"ansiout\">newinstance(class d3.Edge,invoke(input[0, StringType],toString,ObjectType(class java.lang.String)),invoke(input[1, StringType],toString,ObjectType(class java.lang.String)),upcast('count,LongType,- field (class: "scala.Long", name: "count"),- root class: "d3.Edge"),false,ObjectType(class d3.Edge),None)\n:- invoke(input[0, StringType],toString,ObjectType(class java.lang.String))\n: +- input[0, StringType]\n:- invoke(input[1, StringType],toString,ObjectType(class java.lang.String))\n: +- input[1, StringType]\n+- upcast('count,LongType,- field (class: "scala.Long", name: "count"),- root class: "d3.Edge")\n +- 'count\n\n\tat org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.fromRow(ExpressionEncoder.scala:224)\n\tat org.apache.spark.sql.Dataset$$anonfun$collect$2.apply(Dataset.scala:668)\n\tat org.apache.spark.sql.Dataset$$anonfun$collect$2.apply(Dataset.scala:668)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)\n\tat scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)\n\tat scala.collection.TraversableLike$class.map(TraversableLike.scala:244)\n\tat scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:108)\n\tat org.apache.spark.sql.Dataset.collect(Dataset.scala:668)\n\tat d3.graphs$.force(<driver>:19)\nCaused by: java.lang.UnsupportedOperationException: Cannot evaluate expression: upcast('count,LongType,- field (class: "scala.Long", name: "count"),- root class: "d3.Edge")\n\tat org.apache.spark.sql.catalyst.expressions.Unevaluable$class.genCode(Expression.scala:239)\n\tat org.apache.spark.sql.catalyst.expressions.UpCast.genCode(Cast.scala:923)\n\tat org.apache.spark.sql.catalyst.expressions.Expression$$anonfun$gen$2.apply(Expression.scala:104)\n\tat org.apache.spark.sql.catalyst.expressions.Expression$$anonfun$gen$2.apply(Expression.scala:100)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.sql.catalyst.expressions.Expression.gen(Expression.scala:100)\n\tat org.apache.spark.sql.catalyst.expressions.NewInstance$$anonfun$7.apply(objects.scala:219)\n\tat org.apache.spark.sql.catalyst.expressions.NewInstance$$anonfun$7.apply(objects.scala:219)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.immutable.List.foreach(List.scala:318)\n\tat scala.collection.TraversableLike$class.map(TraversableLike.scala:244)\n\tat scala.collection.AbstractTraversable.map(Traversable.scala:105)\n\tat org.apache.spark.sql.catalyst.expressions.NewInstance.genCode(objects.scala:219)\n\tat org.apache.spark.sql.catalyst.expressions.Expression$$anonfun$gen$2.apply(Expression.scala:104)\n\tat org.apache.spark.sql.catalyst.expressions.Expression$$anonfun$gen$2.apply(Expression.scala:100)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.sql.catalyst.expressions.Expression.gen(Expression.scala:100)\n\tat org.apache.spark.sql.catalyst.expressions.codegen.GenerateSafeProjection$$anonfun$3.apply(GenerateSafeProjection.scala:137)\n\tat org.apache.spark.sql.catalyst.expressions.codegen.GenerateSafeProjection$$anonfun$3.apply(GenerateSafeProjection.scala:134)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.immutable.List.foreach(List.scala:318)\n\tat scala.collection.TraversableLike$class.map(TraversableLike.scala:244)\n\tat scala.collection.AbstractTraversable.map(Traversable.scala:105)\n\tat org.apache.spark.sql.catalyst.expressions.codegen.GenerateSafeProjection$.create(GenerateSafeProjection.scala:134)\n\tat org.apache.spark.sql.catalyst.expressions.codegen.GenerateSafeProjection$.create(GenerateSafeProjection.scala:30)\n\tat org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:588)\n\tat org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.constructProjection$lzycompute(ExpressionEncoder.scala:199)\n\tat org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.constructProjection(ExpressionEncoder.scala:199)\n\tat org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.fromRow(ExpressionEncoder.scala:221)\n\tat org.apache.spark.sql.Dataset$$anonfun$collect$2.apply(Dataset.scala:668)\n\tat org.apache.spark.sql.Dataset$$anonfun$collect$2.apply(Dataset.scala:668)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)\n\tat scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)\n\tat scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)\n\tat scala.collection.TraversableLike$class.map(TraversableLike.scala:244)\n\tat scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:108)\n\tat org.apache.spark.sql.Dataset.collect(Dataset.scala:668)\n\tat d3.graphs$.force(<driver>:19)</div>","startTime":1.462424323872E12,"submitTime":1.462424153682E12,"finishTime":1.462424325224E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2a30bd6f-9cea-48a6-a10a-78e9c025fea3"},{"version":"CommandV1","origId":104119,"guid":"0ab13add-a340-4443-991a-429826ffc929","subtype":"command","commandType":"auto","position":4.875,"command":"// This example graph also comes with the GraphFrames package.\nval g0 = examples.Graphs.friends","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">g0: org.graphframes.GraphFrame = GraphFrame(v:[id: string, name: string, age: int], e:[src: string, dst: string, relationship: string])\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.462424353949E12,"submitTime":1.462424183743E12,"finishTime":1.462424354182E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"559a635a-89f5-4e1d-b547-0a0bfeb39636"},{"version":"CommandV1","origId":106382,"guid":"088f5c10-78ea-4acd-88f8-db85251879d1","subtype":"command","commandType":"auto","position":4.9375,"command":"d3.graphs.force( // let us see g0 now in one cell\n height = 500,\n width = 500,\n clicks = g0.edges.select($\"src\", $\"dst\".as(\"dest\"), lit(1L).as(\"count\")).as[d3.Edge])","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"\n<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n <title>Polish Books Themes - an Interactive Map</title>\n <meta charset=\"utf-8\">\n<style>\n\n.node_circle {\n stroke: #777;\n stroke-width: 1.3px;\n}\n\n.node_label {\n pointer-events: none;\n}\n\n.link {\n stroke: #777;\n stroke-opacity: .2;\n}\n\n.node_count {\n stroke: #777;\n stroke-width: 1.0px;\n fill: #999;\n}\n\ntext.legend {\n font-family: Verdana;\n font-size: 13px;\n fill: #000;\n}\n\n.node text {\n font-family: \"Helvetica Neue\",\"Helvetica\",\"Arial\",sans-serif;\n font-size: 17px;\n font-weight: 200;\n}\n\n</style>\n</head>\n\n<body>\n<script src=\"//d3js.org/d3.v3.min.js\"><\/script>\n<script>\n\nvar graph = {\"nodes\":[{\"name\":\"e\"},{\"name\":\"f\"},{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"},{\"name\":\"d\"}],\"links\":[{\"source\":2,\"target\":3,\"value\":1},{\"source\":3,\"target\":4,\"value\":1},{\"source\":4,\"target\":3,\"value\":1},{\"source\":1,\"target\":4,\"value\":1},{\"source\":0,\"target\":1,\"value\":1},{\"source\":0,\"target\":5,\"value\":1},{\"source\":5,\"target\":2,\"value\":1},{\"source\":2,\"target\":0,\"value\":1}]};\n\nvar width = 500,\n height = 500;\n\nvar color = d3.scale.category20();\n\nvar force = d3.layout.force()\n .charge(-700)\n .linkDistance(180)\n .size([width, height]);\n\nvar svg = d3.select(\"body\").append(\"svg\")\n .attr(\"width\", width)\n .attr(\"height\", height);\n \nforce\n .nodes(graph.nodes)\n .links(graph.links)\n .start();\n\nvar link = svg.selectAll(\".link\")\n .data(graph.links)\n .enter().append(\"line\")\n .attr(\"class\", \"link\")\n .style(\"stroke-width\", function(d) { return Math.sqrt(d.value); });\n\nvar node = svg.selectAll(\".node\")\n .data(graph.nodes)\n .enter().append(\"g\")\n .attr(\"class\", \"node\")\n .call(force.drag);\n\nnode.append(\"circle\")\n .attr(\"r\", 10)\n .style(\"fill\", function (d) {\n if (d.name.startsWith(\"other\")) { return color(1); } else { return color(2); };\n})\n\nnode.append(\"text\")\n .attr(\"dx\", 10)\n .attr(\"dy\", \".35em\")\n .text(function(d) { return d.name });\n \n//Now we are giving the SVGs co-ordinates - the force layout is generating the co-ordinates which this code is using to update the attributes of the SVG elements\nforce.on(\"tick\", function () {\n link.attr(\"x1\", function (d) {\n return d.source.x;\n })\n .attr(\"y1\", function (d) {\n return d.source.y;\n })\n .attr(\"x2\", function (d) {\n return d.target.x;\n })\n .attr(\"y2\", function (d) {\n return d.target.y;\n });\n d3.selectAll(\"circle\").attr(\"cx\", function (d) {\n return d.x;\n })\n .attr(\"cy\", function (d) {\n return d.y;\n });\n d3.selectAll(\"text\").attr(\"x\", function (d) {\n return d.x;\n })\n .attr(\"y\", function (d) {\n return d.y;\n });\n});\n<\/script>\n</html>\n","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\"><console>:52: error: not found: value g0\n clicks = g0.edges.select($"src", $"dst".as("dest"), lit(1L).as("count")).as[d3.Edge])\n ^\n</div>","error":null,"startTime":1.462424373771E12,"submitTime":1.462424203578E12,"finishTime":1.4624243742E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"65e2eee9-ab26-45cb-af43-dfaa1e53fb3a"},{"version":"CommandV1","origId":104120,"guid":"74119c75-c3dc-4bd9-8d2c-125fad8b065a","subtype":"command","commandType":"auto","position":5.0,"command":"%md\n## Basic graph and DataFrame queries\n\nGraphFrames provide several simple graph queries, such as node degree.\n\nAlso, since GraphFrames represent graphs as pairs of vertex and edge DataFrames, it is easy to make powerful queries directly on the vertex and edge DataFrames. Those DataFrames are made available as vertices and edges fields in the GraphFrame.\n\n### Simple queries are simple\n\nGraphFrames make it easy to express queries over graphs. Since GraphFrame vertices and edges are stored as DataFrames, many queries are just DataFrame (or SQL) queries.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375237932E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b26c8a43-98f2-4077-ad02-8caf280a7edf"},{"version":"CommandV1","origId":104121,"guid":"91c065da-3c70-4a67-a138-dc2edc80e7fd","subtype":"command","commandType":"auto","position":6.25,"command":"display(g.vertices)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0],["b","Bob",36.0],["c","Charlie",30.0],["d","David",29.0],["e","Esther",32.0],["f","Fanny",36.0],["g","Gabby",60.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462424411006E12,"submitTime":1.462424240778E12,"finishTime":1.462424411205E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"819.77778","height":"205.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"e6b7b63b-d835-48b0-92e8-2a8cc9373603"},{"version":"CommandV1","origId":106350,"guid":"35849966-d41d-46d4-b513-5fdc8f9ba5e0","subtype":"command","commandType":"auto","position":6.5,"command":"display(g0.vertices) // this is the same query on the graph loaded as an example from GraphFrame package","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0],["b","Bob",36.0],["c","Charlie",30.0],["d","David",29.0],["e","Esther",32.0],["f","Fanny",36.0],["g","Gabby",60.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":null,"error":null,"startTime":1.462424425105E12,"submitTime":1.462424254884E12,"finishTime":1.462424425274E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"825.5","height":"183.5","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"da36ca1b-efd6-4ba1-abbd-458df49dac56"},{"version":"CommandV1","origId":104122,"guid":"b843527e-8734-4150-8c8c-1809e2a87d86","subtype":"command","commandType":"auto","position":6.75,"command":"display(g.edges)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","b","friend"],["b","c","follow"],["c","b","follow"],["f","c","follow"],["e","f","follow"],["e","d","friend"],["d","a","friend"],["a","e","friend"]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"src","type":"\"string\""},{"name":"dst","type":"\"string\""},{"name":"relationship","type":"\"string\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462424430664E12,"submitTime":1.462424260451E12,"finishTime":1.462424430787E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"814.77778","height":"204.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2e4a5945-8ad4-42d8-b167-7fcc2c346bdf"},{"version":"CommandV1","origId":104123,"guid":"43992b81-0bbb-40a2-8b17-555cb24ed6eb","subtype":"command","commandType":"auto","position":7.5,"command":"%md \nThe incoming degree of the vertices:","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"aa29ad84-784e-4b0a-800b-28620b4da439"},{"version":"CommandV1","origId":104124,"guid":"a3455f7a-9da5-4cd7-882d-dab2f0497ec4","subtype":"command","commandType":"auto","position":8.25,"command":"display(g.inDegrees)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a",1.0],["b",2.0],["c",2.0],["d",1.0],["e",1.0],["f",1.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"inDegree","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462424445275E12,"submitTime":1.462424275085E12,"finishTime":1.462424449815E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"829.77778","height":"167.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"872fc09d-89da-4ec0-aa33-02bd7f08e4a1"},{"version":"CommandV1","origId":104125,"guid":"3c0b2f26-6e7a-4f19-b004-0cbf2f80acce","subtype":"command","commandType":"auto","position":8.5,"command":"%md \nThe outgoing degree of the vertices:","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"07cd7613-d39d-4fcf-a6cb-6db61efec590"},{"version":"CommandV1","origId":104126,"guid":"40fbb7ab-348b-4c6d-8596-8367beca128b","subtype":"command","commandType":"auto","position":8.75,"command":"display(g.outDegrees)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a",2.0],["b",1.0],["c",1.0],["d",1.0],["e",2.0],["f",1.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"outDegree","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462424457478E12,"submitTime":1.462424287271E12,"finishTime":1.462424459246E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c5180419-e89b-4202-9319-9f75f15cb3c3"},{"version":"CommandV1","origId":104127,"guid":"f02deb4b-81fa-4bb4-bdcc-5723d246137b","subtype":"command","commandType":"auto","position":8.875,"command":"%md \nThe degree of the vertices:","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7c4ab866-d495-48bb-97c7-df8dcddd60c6"},{"version":"CommandV1","origId":104128,"guid":"c0e7db9d-928c-421f-a9b9-c79c30a780de","subtype":"command","commandType":"auto","position":9.0,"command":"display(g.degrees)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a",3.0],["b",3.0],["c",3.0],["d",2.0],["e",3.0],["f",2.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"degree","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462424464271E12,"submitTime":1.462424294066E12,"finishTime":1.462424465722E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d6433f17-8361-4061-a505-4c778b70cf89"},{"version":"CommandV1","origId":104129,"guid":"b21a720f-627d-4475-b5ea-32c3e710b537","subtype":"command","commandType":"auto","position":9.125,"command":"%md \nYou can run queries directly on the vertices DataFrame. For example, we can find the age of the youngest person in the graph:","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"97c8f12b-3b15-453e-9e8a-f2256a9595be"},{"version":"CommandV1","origId":104130,"guid":"4fc425ef-c053-4295-a17f-756aa4d11d5f","subtype":"command","commandType":"auto","position":9.25,"command":"val youngest = g.vertices.groupBy().min(\"age\")\ndisplay(youngest)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[29.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"min(age)","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462424485668E12,"submitTime":1.462424315469E12,"finishTime":1.462424486223E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"408.5","height":"109.5","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9dc113c4-254d-4db7-a9f9-1b6039b0b6f8"},{"version":"CommandV1","origId":104131,"guid":"49e831cf-2b41-44d1-83e7-014aabc0d6bb","subtype":"command","commandType":"auto","position":9.75,"command":"%md \nLikewise, you can run queries on the edges DataFrame. \n\nFor example, let us count the number of 'follow' relationships in the graph:","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"bcacdbb7-2a09-401d-8f61-19d87ff5b240"},{"version":"CommandV1","origId":104132,"guid":"6fe284e3-bb79-4fd5-aa42-ea09fe96b07c","subtype":"command","commandType":"auto","position":10.25,"command":"val numFollows = g.edges.filter(\"relationship = 'follow'\").count()","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">numFollows: Long = 4\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.462424512592E12,"submitTime":1.462424342399E12,"finishTime":1.462424513122E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"01843d50-ee08-43c2-9070-2ddbddb15f34"},{"version":"CommandV1","origId":104133,"guid":"8b0cdeab-256b-4c43-8c6e-0174099562fe","subtype":"command","commandType":"auto","position":11.25,"command":"%md \n##Motif finding\n\nMore complex relationships involving edges and vertices can be built using motifs. \n\nThe following cell finds the pairs of vertices with edges in both directions between them. \n\nThe result is a dataframe, in which the column names are given by the motif keys.\n\nCheck out the [GraphFrame User Guide](http://graphframes.github.io/user-guide.html#motif-finding) for more details on the API.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375239295E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b6fd88d8-98c4-4c6f-92db-c4e7cde27ab4"},{"version":"CommandV1","origId":104134,"guid":"46fc26d4-dcdf-44eb-9580-d7c82b424007","subtype":"command","commandType":"auto","position":12.25,"command":"// Search for pairs of vertices with edges in both directions between them, i.e., find undirected or bidirected edges.\nval motifs = g.find(\"(a)-[e1]->(b); (b)-[e2]->(a)\")\ndisplay(motifs)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[["b","c","follow"],["b","Bob",36.0],["c","Charlie",30.0],["c","b","follow"]],[["c","b","follow"],["c","Charlie",30.0],["b","Bob",36.0],["b","c","follow"]]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"e1","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"a","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"b","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"e2","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462424596011E12,"submitTime":1.462424425822E12,"finishTime":1.462424596922E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"1037","height":"170","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"a573fbd9-458b-43f5-89ea-27f5f6253ef3"},{"version":"CommandV1","origId":104135,"guid":"1e12da86-f0a1-4dea-a9f0-dd1ee9e328b1","subtype":"command","commandType":"auto","position":12.75,"command":"%md \nSince the result is a DataFrame, more complex queries can be built on top of the motif. \n\nLet us find all the reciprocal relationships in which one person is older than 30:","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"875aa251-c0a3-42d3-a8bc-373f2c5b6fd4"},{"version":"CommandV1","origId":104136,"guid":"1e0e1f22-30e2-4662-8d55-c5e3e012126a","subtype":"command","commandType":"auto","position":13.25,"command":"val filtered = motifs.filter(\"b.age > 30\")\ndisplay(filtered)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[["c","b","follow"],["c","Charlie",30.0],["b","Bob",36.0],["b","c","follow"]]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"e1","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"a","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"b","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"e2","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"<div class=\"ansiout\"><console>:41: error: not found: value motifs\n val filtered = motifs.filter("b.age > 30")\n ^\n</div>","error":null,"startTime":1.462424823942E12,"submitTime":1.462424653718E12,"finishTime":1.462424824615E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"1235.77778","height":"137.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"a7b92e85-8980-42e5-9d75-0845bfddf2aa"},{"version":"CommandV1","origId":106360,"guid":"8baf6cd3-a0c3-4295-ab6c-7da060216cf4","subtype":"command","commandType":"auto","position":13.5,"command":"%md\n**You Try!**\n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b2c2b1b8-9b97-4c02-be7a-86bcdc794ff6"},{"version":"CommandV1","origId":106362,"guid":"3f818eee-f239-4add-b4bf-ef202d074529","subtype":"command","commandType":"auto","position":13.625,"command":"//Search for all \"directed triangles\" or triplets of vertices: a,b,c with edges: a->b, b->c and c->a\n//uncomment the next 2 lines and replace the \"...\" below\nval motifs3 = g.find(\"(a)-[e1]->(b); (b)-[e2]->(c); (c)-[e3]->(a) \")\ndisplay(motifs3)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[["e","d","friend"],["e","Esther",32.0],["d","David",29.0],["d","a","friend"],["a","Alice",34.0],["a","e","friend"]],[["d","a","friend"],["d","David",29.0],["a","Alice",34.0],["a","e","friend"],["e","Esther",32.0],["e","d","friend"]],[["a","e","friend"],["a","Alice",34.0],["e","Esther",32.0],["e","d","friend"],["d","David",29.0],["d","a","friend"]]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"e1","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"a","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"b","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"e2","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"c","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"e3","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"org.graphframes.InvalidParseException: Failed to parse bad motif string: '(a)-[e1]->(b); (b)-[e2]->(c); (c)-[e3]->(a);'. Returned message: `!' expected but end of source found","error":"<div class=\"ansiout\">\tat org.graphframes.pattern.Pattern$.parse(patterns.scala:55)\n\tat org.graphframes.GraphFrame.find(GraphFrame.scala:263)</div>","startTime":1.462424886668E12,"submitTime":1.462424716452E12,"finishTime":1.462424887529E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f96b06f3-e7cc-4497-9d91-47ce9cc4144c"},{"version":"CommandV1","origId":104137,"guid":"dfff1061-4822-4dd3-8a3c-6bffc6fd43f7","subtype":"command","commandType":"auto","position":13.75,"command":"%md\n**Stateful queries**\n\nMany motif queries are stateless and simple to express, as in the examples above. The next examples demonstrate more complex queries which carry state along a path in the motif. These queries can be expressed by combining GraphFrame motif finding with filters on the result, where the filters use sequence operations to construct a series of DataFrame Columns.\n\nFor example, suppose one wishes to identify a chain of 4 vertices with some property defined by a sequence of functions. That is, among chains of 4 vertices `a->b->c->d`, identify the subset of chains matching this complex filter:\n\n* Initialize state on path.\n* Update state based on vertex a.\n* Update state based on vertex b.\n* Etc. for c and d.\n* If final state matches some condition, then the chain is accepted by the filter.\n\nThe below code snippets demonstrate this process, where we identify chains of 4 vertices such that at least 2 of the 3 edges are `friend` relationships. In this example, the state is the current count of `friend` edges; in general, it could be any DataFrame Column.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.45637523981E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2455831e-b504-4186-9fd7-11e08f68cc93"},{"version":"CommandV1","origId":104138,"guid":"8e69bf0e-dd13-4056-b288-f6a0a7b6df44","subtype":"command","commandType":"auto","position":14.25,"command":"// Find chains of 4 vertices.\nval chain4 = g.find(\"(a)-[ab]->(b); (b)-[bc]->(c); (c)-[cd]->(d)\")\n\n// Query on sequence, with state (cnt)\n// (a) Define method for updating state given the next element of the motif.\ndef sumFriends(cnt: Column, relationship: Column): Column = {\n when(relationship === \"friend\", cnt + 1).otherwise(cnt)\n}\n// (b) Use sequence operation to apply method to sequence of elements in motif.\n// In this case, the elements are the 3 edges.\nval condition = Seq(\"ab\", \"bc\", \"cd\").\n foldLeft(lit(0))((cnt, e) => sumFriends(cnt, col(e)(\"relationship\")))\n// (c) Apply filter to DataFrame.\nval chainWith2Friends2 = chain4.where(condition >= 2)\ndisplay(chainWith2Friends2)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[["e","d","friend"],["e","Esther",32.0],["d","David",29.0],["d","a","friend"],["a","Alice",34.0],["a","b","friend"],["b","Bob",36.0]],[["e","d","friend"],["e","Esther",32.0],["d","David",29.0],["d","a","friend"],["a","Alice",34.0],["a","e","friend"],["e","Esther",32.0]],[["d","a","friend"],["d","David",29.0],["a","Alice",34.0],["a","b","friend"],["b","Bob",36.0],["b","c","follow"],["c","Charlie",30.0]],[["d","a","friend"],["d","David",29.0],["a","Alice",34.0],["a","e","friend"],["e","Esther",32.0],["e","f","follow"],["f","Fanny",36.0]],[["d","a","friend"],["d","David",29.0],["a","Alice",34.0],["a","e","friend"],["e","Esther",32.0],["e","d","friend"],["d","David",29.0]],[["a","e","friend"],["a","Alice",34.0],["e","Esther",32.0],["e","d","friend"],["d","David",29.0],["d","a","friend"],["a","Alice",34.0]]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"ab","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"a","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"b","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"bc","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"c","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"cd","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"d","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462425215568E12,"submitTime":1.462425045248E12,"finishTime":1.462425216732E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f6dd9e87-79eb-47c3-a905-4cccba248980"},{"version":"CommandV1","origId":104139,"guid":"7f74dc78-ed80-4469-a937-e08bdee7014b","subtype":"command","commandType":"auto","position":15.25,"command":"%md\n###Subgraphs\n\nSubgraphs are built by filtering a subset of edges and vertices. For example, the following subgraph only contains people who are friends and who are more than 30 years old.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375240128E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"104e3c55-b9ce-4861-ba1e-3ec62b9b739f"},{"version":"CommandV1","origId":104140,"guid":"82956d1a-202f-4c84-96fd-e63c63a1df36","subtype":"command","commandType":"auto","position":16.25,"command":"// Select subgraph of users older than 30, and edges of type \"friend\"\nval v2 = g.vertices.filter(\"age > 30\")\nval e2 = g.edges.filter(\"relationship = 'friend'\")\nval g2 = GraphFrame(v2, e2)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">v2: org.apache.spark.sql.DataFrame = [id: string, name: string, age: int]\ne2: org.apache.spark.sql.DataFrame = [src: string, dst: string, relationship: string]\ng2: org.graphframes.GraphFrame = GraphFrame(v:[id: string, name: string, age: int], e:[src: string, dst: string, relationship: string])\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.462425553536E12,"submitTime":1.462425383292E12,"finishTime":1.462425553647E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c587e1df-4d8a-4dce-a38d-109e180e696d"},{"version":"CommandV1","origId":104141,"guid":"3d6bea4a-7fb7-4972-9c28-e20ce2bdf993","subtype":"command","commandType":"auto","position":17.25,"command":"display(g2.vertices)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0],["b","Bob",36.0],["e","Esther",32.0],["f","Fanny",36.0],["g","Gabby",60.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462425623604E12,"submitTime":1.462425453365E12,"finishTime":1.462425623806E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"597.77778","height":"203.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9415c252-f425-4d1b-a035-a225cc2b93e3"},{"version":"CommandV1","origId":104142,"guid":"5d5ca99c-b70e-4063-bbc2-9def0be46d65","subtype":"command","commandType":"auto","position":18.25,"command":"display(g2.edges)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","b","friend"],["e","d","friend"],["d","a","friend"],["a","e","friend"]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"src","type":"\"string\""},{"name":"dst","type":"\"string\""},{"name":"relationship","type":"\"string\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462425628304E12,"submitTime":1.46242545806E12,"finishTime":1.462425628532E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"580.77778","height":"201.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c33abcb0-e48b-4d02-abc0-90f491c56760"},{"version":"CommandV1","origId":106383,"guid":"2196bd72-ee50-4e72-b4fd-3b6cbde02259","subtype":"command","commandType":"auto","position":18.75,"command":"d3.graphs.force( // let us see g2 now in one cell\n height = 500,\n width = 500,\n clicks = g2.edges.select($\"src\", $\"dst\".as(\"dest\"), lit(1L).as(\"count\")).as[d3.Edge])","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"\n<!DOCTYPE html>\n<html>\n<head>\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n <title>Polish Books Themes - an Interactive Map</title>\n <meta charset=\"utf-8\">\n<style>\n\n.node_circle {\n stroke: #777;\n stroke-width: 1.3px;\n}\n\n.node_label {\n pointer-events: none;\n}\n\n.link {\n stroke: #777;\n stroke-opacity: .2;\n}\n\n.node_count {\n stroke: #777;\n stroke-width: 1.0px;\n fill: #999;\n}\n\ntext.legend {\n font-family: Verdana;\n font-size: 13px;\n fill: #000;\n}\n\n.node text {\n font-family: \"Helvetica Neue\",\"Helvetica\",\"Arial\",sans-serif;\n font-size: 17px;\n font-weight: 200;\n}\n\n</style>\n</head>\n\n<body>\n<script src=\"//d3js.org/d3.v3.min.js\"><\/script>\n<script>\n\nvar graph = {\"nodes\":[{\"name\":\"a\"},{\"name\":\"e\"},{\"name\":\"d\"},{\"name\":\"b\"}],\"links\":[{\"source\":0,\"target\":3,\"value\":1},{\"source\":1,\"target\":2,\"value\":1},{\"source\":2,\"target\":0,\"value\":1},{\"source\":0,\"target\":1,\"value\":1}]};\n\nvar width = 500,\n height = 500;\n\nvar color = d3.scale.category20();\n\nvar force = d3.layout.force()\n .charge(-700)\n .linkDistance(180)\n .size([width, height]);\n\nvar svg = d3.select(\"body\").append(\"svg\")\n .attr(\"width\", width)\n .attr(\"height\", height);\n \nforce\n .nodes(graph.nodes)\n .links(graph.links)\n .start();\n\nvar link = svg.selectAll(\".link\")\n .data(graph.links)\n .enter().append(\"line\")\n .attr(\"class\", \"link\")\n .style(\"stroke-width\", function(d) { return Math.sqrt(d.value); });\n\nvar node = svg.selectAll(\".node\")\n .data(graph.nodes)\n .enter().append(\"g\")\n .attr(\"class\", \"node\")\n .call(force.drag);\n\nnode.append(\"circle\")\n .attr(\"r\", 10)\n .style(\"fill\", function (d) {\n if (d.name.startsWith(\"other\")) { return color(1); } else { return color(2); };\n})\n\nnode.append(\"text\")\n .attr(\"dx\", 10)\n .attr(\"dy\", \".35em\")\n .text(function(d) { return d.name });\n \n//Now we are giving the SVGs co-ordinates - the force layout is generating the co-ordinates which this code is using to update the attributes of the SVG elements\nforce.on(\"tick\", function () {\n link.attr(\"x1\", function (d) {\n return d.source.x;\n })\n .attr(\"y1\", function (d) {\n return d.source.y;\n })\n .attr(\"x2\", function (d) {\n return d.target.x;\n })\n .attr(\"y2\", function (d) {\n return d.target.y;\n });\n d3.selectAll(\"circle\").attr(\"cx\", function (d) {\n return d.x;\n })\n .attr(\"cy\", function (d) {\n return d.y;\n });\n d3.selectAll(\"text\").attr(\"x\", function (d) {\n return d.x;\n })\n .attr(\"y\", function (d) {\n return d.y;\n });\n});\n<\/script>\n</html>\n","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462425633991E12,"submitTime":1.462425463749E12,"finishTime":1.462425634504E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"0096b355-3bfe-4600-bd33-8430bdbb46c4"},{"version":"CommandV1","origId":104143,"guid":"fee94893-3878-41a8-ac2b-94af62bb852c","subtype":"command","commandType":"auto","position":19.25,"command":"%md \n**Complex triplet filters**\n\nThe following example shows how to select a subgraph based upon triplet filters which operate on:\n* an edge and \n* its src and \n* dst vertices. \n\nThis example could be extended to go beyond triplets by using more complex motifs.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375241027E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d21f75b8-603e-4b82-9c29-16c66322a21c"},{"version":"CommandV1","origId":104144,"guid":"217f8562-e495-4e6f-96dc-c197500ffde2","subtype":"command","commandType":"auto","position":20.25,"command":"// Select subgraph based on edges \"e\" of type \"follow\"\n// pointing from a younger user \"a\" to an older user \"b\".\nval paths = g.find(\"(a)-[e]->(b)\")\n .filter(\"e.relationship = 'follow'\")\n .filter(\"a.age < b.age\")\n// \"paths\" contains vertex info. Extract the edges.\nval e2 = paths.select(\"e.src\", \"e.dst\", \"e.relationship\")\n// In Spark 1.5+, the user may simplify this call:\n// val e2 = paths.select(\"e.*\")\n\n// Construct the subgraph\nval g2 = GraphFrame(g.vertices, e2)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">paths: org.apache.spark.sql.DataFrame = [e: struct<src:string,dst:string,relationship:string>, a: struct<id:string,name:string,age:int>, b: struct<id:string,name:string,age:int>]\ne2: org.apache.spark.sql.DataFrame = [src: string, dst: string, relationship: string]\ng2: org.graphframes.GraphFrame = GraphFrame(v:[id: string, name: string, age: int], e:[src: string, dst: string, relationship: string])\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.462425730673E12,"submitTime":1.462425560428E12,"finishTime":1.462425730897E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"fc71a4a0-4da0-48ca-84a8-ad9073418205"},{"version":"CommandV1","origId":104145,"guid":"2539bcfa-d55f-4367-85c0-ab3640dbddd4","subtype":"command","commandType":"auto","position":21.25,"command":"display(g2.vertices)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0],["b","Bob",36.0],["c","Charlie",30.0],["d","David",29.0],["e","Esther",32.0],["f","Fanny",36.0],["g","Gabby",60.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462425735388E12,"submitTime":1.462425565101E12,"finishTime":1.462425735497E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"871.77778","height":"240.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"e2bf94db-49da-49a2-94d5-8d0c68e464c3"},{"version":"CommandV1","origId":104146,"guid":"b686f739-115d-4202-9664-fcae2aaa9404","subtype":"command","commandType":"auto","position":22.25,"command":"display(g2.edges)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["c","b","follow"],["e","f","follow"]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"src","type":"\"string\""},{"name":"dst","type":"\"string\""},{"name":"relationship","type":"\"string\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462425747919E12,"submitTime":1.462425577654E12,"finishTime":1.462425748383E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"894.77778","height":"121.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2c89f166-1f10-4067-9069-e76fde7329b3"},{"version":"CommandV1","origId":104147,"guid":"969eab9f-6236-4e5e-8049-4260e9431ab4","subtype":"command","commandType":"auto","position":23.25,"command":"%md \n## Standard graph algorithms\n\nGraphFrames comes with a number of standard graph algorithms built in:\n\n* Breadth-first search (BFS)\n* Connected components\n* Strongly connected components\n* Label Propagation Algorithm (LPA)\n* PageRank\n* Shortest paths\n* Triangle count","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375241686E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"720204d3-4fa2-4af0-9bfa-fbd7cceb4ffa"},{"version":"CommandV1","origId":104148,"guid":"67d42e07-178d-4c38-bc95-f0c41927191c","subtype":"command","commandType":"auto","position":24.25,"command":"%md \n### Breadth-first search (BFS)\n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375241873E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cbdcd4a8-dfd3-4566-83ca-65e556ee772d"},{"version":"CommandV1","origId":106386,"guid":"511222ff-5cf2-487e-b7c9-d4b37677a8e3","subtype":"command","commandType":"auto","position":24.875,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#breadth-first-search-bfs\",500))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#breadth-first-search-bfs\"\n width=\"95%\" height=\"500\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462417009946E12,"submitTime":1.462416840073E12,"finishTime":1.462417010089E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"6a622ffa-f71b-46f1-b2a2-0e3052ba7714"},{"version":"CommandV1","origId":106385,"guid":"0a4f1ebb-bec7-4c6e-bce0-ea602a8d056c","subtype":"command","commandType":"auto","position":25.0,"command":"%md\nSearch from \"Esther\" for users of age < 32.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7c30609a-3a86-4f7f-864b-1f1c7b00d5a7"},{"version":"CommandV1","origId":104149,"guid":"9676ad0d-7966-4682-a376-9d52bc0c8596","subtype":"command","commandType":"auto","position":25.25,"command":"val paths: DataFrame = g.bfs.fromExpr(\"name = 'Esther'\").toExpr(\"age < 32\").run()\ndisplay(paths)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[["e","Esther",32.0],["e","d","friend"],["d","David",29.0]]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"from","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"e0","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"to","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462425937764E12,"submitTime":1.462425767522E12,"finishTime":1.462425938759E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"1040.77778","height":"119.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"e065156e-37c7-4dbb-84a1-d7e10fe47db4"},{"version":"CommandV1","origId":104150,"guid":"0566c152-d201-4e02-b6f6-344adc487f92","subtype":"command","commandType":"auto","position":25.75,"command":"%md \nThe search may also be limited by edge filters and maximum path lengths.","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"264d4d7b-ab61-49ec-b4b1-e7f7178c9188"},{"version":"CommandV1","origId":104151,"guid":"773c1fb4-20ef-421a-86d2-ddb2df9d1303","subtype":"command","commandType":"auto","position":26.25,"command":"val filteredPaths = g.bfs.fromExpr(\"name = 'Esther'\").toExpr(\"age < 32\")\n .edgeFilter(\"relationship != 'friend'\")\n .maxPathLength(3)\n .run()\ndisplay(filteredPaths)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[["e","Esther",32.0],["e","f","follow"],["f","Fanny",36.0],["f","c","follow"],["c","Charlie",30.0]]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"from","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"e0","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"v1","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"e1","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"src\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"dst\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"relationship\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}"},{"name":"to","type":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"age\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]}"}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462426017142E12,"submitTime":1.46242584687E12,"finishTime":1.462426022814E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"1232.77778","height":"158.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"6422598a-de67-49e1-be48-64e459da4258"},{"version":"CommandV1","origId":104152,"guid":"0a053f5c-ba7d-49aa-979c-3ba3c70c7500","subtype":"command","commandType":"auto","position":27.25,"command":"%md \n###Connected components\n\nCompute the connected component membership of each vertex and return a graph with each vertex assigned a component ID.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375242333E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"22efded5-34c6-4364-8d72-897e2efc6f42"},{"version":"CommandV1","origId":106387,"guid":"d040625c-149b-489f-9f6c-13bf7c363fce","subtype":"command","commandType":"auto","position":27.75,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#connected-components\",500))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#connected-components\"\n width=\"95%\" height=\"500\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462417336071E12,"submitTime":1.462417166167E12,"finishTime":1.462417336192E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c44bc054-5aca-4da7-aa7e-8bd6e73f0ac3"},{"version":"CommandV1","origId":104153,"guid":"76d1e610-f986-4942-9712-36e0f707a111","subtype":"command","commandType":"auto","position":28.25,"command":"val result = g.connectedComponents.run() // doesn't work on Spark 1.4\ndisplay(result)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0,0.0],["d","David",29.0,0.0],["b","Bob",36.0,0.0],["e","Esther",32.0,0.0],["c","Charlie",30.0,0.0],["f","Fanny",36.0,0.0],["g","Gabby",60.0,7.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""},{"name":"component","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462426083194E12,"submitTime":1.462425912936E12,"finishTime":1.462426095302E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"905.77778","height":"255.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"66d569ed-0a82-4933-83f9-5d75dcbc481c"},{"version":"CommandV1","origId":106388,"guid":"ba54e081-e6c0-4adc-b144-45cbebdd97ff","subtype":"command","commandType":"auto","position":28.75,"command":"%md\nFun Exercise: Try to modify the d3.graph function to allow a visualisation of a given Sequence of `component` ids in the above `result`.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"0bd6011d-26d0-45bd-9eac-c8707ff3e900"},{"version":"CommandV1","origId":104154,"guid":"7aead4b2-0e93-41b4-8f5b-146c7b7efd3e","subtype":"command","commandType":"auto","position":29.25,"command":"%md \n##Strongly connected components\n\nCompute the strongly connected component (SCC) of each vertex and return a graph with each vertex assigned to the SCC containing that vertex.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.45637524266E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9b43b97a-4e3a-419b-824e-b1baa9d8303f"},{"version":"CommandV1","origId":106389,"guid":"4dfda5f3-3357-4c3d-bb9d-0bf9af830f71","subtype":"command","commandType":"auto","position":29.75,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#strongly-connected-components\",500))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#strongly-connected-components\"\n width=\"95%\" height=\"500\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462417599437E12,"submitTime":1.462417429518E12,"finishTime":1.462417599546E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d60ca2d3-9b6e-4ce8-9310-1eee3e903a29"},{"version":"CommandV1","origId":104155,"guid":"0a6e2402-69e5-4089-b6b7-22e88f3fbf5a","subtype":"command","commandType":"auto","position":30.25,"command":"val result = g.stronglyConnectedComponents.maxIter(10).run()\ndisplay(result.orderBy(\"component\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["e","Esther",32.0,0.0],["d","David",29.0,0.0],["a","Alice",34.0,0.0],["b","Bob",36.0,2.0],["c","Charlie",30.0,2.0],["f","Fanny",36.0,5.0],["g","Gabby",60.0,7.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""},{"name":"component","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462417698863E12,"submitTime":1.462417528974E12,"finishTime":1.462417731953E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"862.77778","height":"207.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"5d2ece00-8bdd-417a-99be-6ca190bdadd0"},{"version":"CommandV1","origId":104156,"guid":"74023006-2b6f-4d10-83b0-136f18f428b8","subtype":"command","commandType":"auto","position":31.25,"command":"%md \n##Label propagation\n\nRun static Label Propagation Algorithm for detecting communities in networks.\n\nEach node in the network is initially assigned to its own community. At every superstep, nodes send their community affiliation to all neighbors and update their state to the mode community affiliation of incoming messages.\n\nLPA is a standard community detection algorithm for graphs. It is very inexpensive computationally, although \n* (1) convergence is not guaranteed and \n* (2) one can end up with trivial solutions (all nodes are identified into a single community).","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.45637524305E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"31ecdc1b-ecef-4207-bb08-da65d87bd196"},{"version":"CommandV1","origId":106390,"guid":"c2fdb369-2516-428d-a4e5-84b39dc379bf","subtype":"command","commandType":"auto","position":31.75,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#label-propagation-algorithm-lpa\",600))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#label-propagation-algorithm-lpa\"\n width=\"95%\" height=\"600\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462417928522E12,"submitTime":1.462417758612E12,"finishTime":1.462417928642E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f89638f1-0303-4440-a1c4-8cec6c1dd4cd"},{"version":"CommandV1","origId":104157,"guid":"06945f17-9202-4961-8d8e-b0b48b282b95","subtype":"command","commandType":"auto","position":32.25,"command":"val result = g.labelPropagation.maxIter(5).run()\ndisplay(result.orderBy(\"label\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0,2.0],["c","Charlie",30.0,2.0],["e","Esther",32.0,2.0],["b","Bob",36.0,4.0],["d","David",29.0,4.0],["f","Fanny",36.0,4.0],["g","Gabby",60.0,7.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""},{"name":"label","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"<div class=\"ansiout\"><console>:53: error: value vertices is not a member of org.apache.spark.sql.DataFrame\n display(result.vertices.orderBy("label"))\n ^\n</div>","error":null,"startTime":1.462426413362E12,"submitTime":1.462426243083E12,"finishTime":1.462426424564E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"828.77778","height":"215.77778","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b8ab30f3-9686-4e04-84ce-ee8ef1e824f0"},{"version":"CommandV1","origId":104158,"guid":"5b03214a-022b-4195-834c-caf9afa83deb","subtype":"command","commandType":"auto","position":33.25,"command":"%md \n##PageRank\n\nIdentify important vertices in a graph based on connections.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375243247E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"584a4a4f-a94e-4933-8e06-318b7e146eeb"},{"version":"CommandV1","origId":106391,"guid":"0635cc34-9647-4e9a-b7a0-413ad717b105","subtype":"command","commandType":"auto","position":33.75,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#pagerank\",600))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#pagerank\"\n width=\"95%\" height=\"600\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462418579414E12,"submitTime":1.462418409443E12,"finishTime":1.462418579519E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cd6a64a0-e31d-4e2e-9ec8-f37f8d5616cf"},{"version":"CommandV1","origId":104159,"guid":"28f95230-a543-4742-936f-f56910be6024","subtype":"command","commandType":"auto","position":34.25,"command":"// Run PageRank until convergence to tolerance \"tol\".\nval results = g.pageRank.resetProbability(0.15).tol(0.01).run()\ndisplay(results.vertices)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0,0.37429242187499995],["d","David",29.0,0.27366105468749996],["b","Bob",36.0,2.2131428039184433],["e","Esther",32.0,0.309074279296875],["c","Charlie",30.0,2.240080617201845],["f","Fanny",36.0,0.27366105468749996],["g","Gabby",60.0,0.15]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""},{"name":"pagerank","type":"\"double\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462426477895E12,"submitTime":1.462426307608E12,"finishTime":1.462426518432E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"06d44d42-3a87-46bc-804e-c28c9263dd51"},{"version":"CommandV1","origId":104160,"guid":"a5d8cddd-5651-48bc-be6c-6f402174b588","subtype":"command","commandType":"auto","position":35.0,"command":"display(results.edges)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["c","b","follow",1.0],["f","c","follow",1.0],["a","b","friend",0.5],["a","e","friend",0.5],["d","a","friend",1.0],["b","c","follow",1.0],["e","d","friend",0.5],["e","f","follow",0.5]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"src","type":"\"string\""},{"name":"dst","type":"\"string\""},{"name":"relationship","type":"\"string\""},{"name":"weight","type":"\"double\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.462419280267E12,"submitTime":1.462419110286E12,"finishTime":1.462419284736E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"4fb776b6-a4c0-4b52-a6f2-4aed2ff1c9b1"},{"version":"CommandV1","origId":104161,"guid":"9d84188d-23cc-45e0-984a-c05ac201e306","subtype":"command","commandType":"auto","position":35.25,"command":"// Run PageRank for a fixed number of iterations.\nval results2 = g.pageRank.resetProbability(0.15).maxIter(10).run()\ndisplay(results2.vertices)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0,0.39143465933514154],["b","Bob",36.0,1.842259190054981],["d","David",29.0,0.28427148788098855],["f","Fanny",36.0,0.28427148788098855],["c","Charlie",30.0,1.877540087856477],["e","Esther",32.0,0.31616362485373634],["g","Gabby",60.0,0.15]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""},{"name":"pagerank","type":"\"double\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.456862938586E12,"submitTime":1.456862938502E12,"finishTime":1.456862946626E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"joseph","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2941bf9d-e278-4ae7-a9e0-93991250b8ce"},{"version":"CommandV1","origId":104162,"guid":"aab4717e-7774-46ed-ba67-b189a0bade99","subtype":"command","commandType":"auto","position":36.25,"command":"// Run PageRank personalized for vertex \"a\"\nval results3 = g.pageRank.resetProbability(0.15).maxIter(10).sourceId(\"a\").run()\ndisplay(results3.vertices)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0,0.1771083164268356],["b","Bob",36.0,0.2699384803126761],["d","David",29.0,0.0318921369727478],["f","Fanny",36.0,0.0318921369727478],["c","Charlie",30.0,0.24655465114397318],["e","Esther",32.0,0.07527103448140514],["g","Gabby",60.0,0.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""},{"name":"pagerank","type":"\"double\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.456862946633E12,"submitTime":1.456862941108E12,"finishTime":1.456862954494E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"joseph","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"121bfce7-fd90-457f-b2ac-089fba1a0d07"},{"version":"CommandV1","origId":104163,"guid":"ab4cc4ee-c1a2-4848-a7d8-c430f8fd3fa9","subtype":"command","commandType":"auto","position":37.25,"command":"%md \n##Shortest paths\n\nComputes shortest paths to the given set of landmark vertices, where landmarks are specified by vertex ID.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375243641E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"aafb9806-5cc4-43e0-abef-8d2d14000cf5"},{"version":"CommandV1","origId":106392,"guid":"a4f7f985-7cad-4fc4-a386-a539ea8b463b","subtype":"command","commandType":"auto","position":37.75,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#shortest-paths\",500))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#shortest-paths\"\n width=\"95%\" height=\"500\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462418881932E12,"submitTime":1.46241871198E12,"finishTime":1.462418882032E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"8958a5a6-bcae-4370-b953-431d52bf5486"},{"version":"CommandV1","origId":104164,"guid":"0a978cfa-a594-44b4-b83d-9c820c05bae1","subtype":"command","commandType":"auto","position":38.25,"command":"val paths = g.shortestPaths.landmarks(Seq(\"a\", \"d\")).run()\ndisplay(paths)","commandVersion":0,"state":"finished","results":{"type":"table","data":[["a","Alice",34.0,{"a":0.0,"d":2.0}],["b","Bob",36.0,{}],["d","David",29.0,{"d":0.0,"a":1.0}],["f","Fanny",36.0,{}],["c","Charlie",30.0,{}],["e","Esther",32.0,{"d":1.0,"a":2.0}],["g","Gabby",60.0,{}]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""},{"name":"distances","type":"{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"integer\",\"valueContainsNull\":true}"}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.4568629623E12,"submitTime":1.456862962199E12,"finishTime":1.456862964895E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"joseph","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"356a6a31-cade-4cbc-8f92-812d8a53074f"},{"version":"CommandV1","origId":104165,"guid":"365bdc5f-ebe0-4db7-83a5-fef9fac7a307","subtype":"command","commandType":"auto","position":39.25,"command":"%md \n###Triangle count\n\nComputes the number of triangles passing through each vertex.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.456375243735E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"tim","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"5011bc37-f0d5-4ee2-b3c2-d4441e12bbfb"},{"version":"CommandV1","origId":106393,"guid":"ae81f7d8-78ab-4e76-986a-276eb0e09826","subtype":"command","commandType":"auto","position":39.75,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#triangle-count\",300))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#triangle-count\"\n width=\"95%\" height=\"300\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.462418973506E12,"submitTime":1.46241880355E12,"finishTime":1.462418973639E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9236b945-223f-42bd-b916-cdeddc1738b8"},{"version":"CommandV1","origId":104166,"guid":"6f289442-ad70-4f38-9e99-691a394b7357","subtype":"command","commandType":"auto","position":40.25,"command":"val results = g.triangleCount.run()\ndisplay(results)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[1.0,"a","Alice",34.0],[0.0,"b","Bob",36.0],[0.0,"c","Charlie",30.0],[1.0,"d","David",29.0],[1.0,"e","Esther",32.0],[0.0,"f","Fanny",36.0],[0.0,"g","Gabby",60.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"count","type":"\"long\""},{"name":"id","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"age","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.456862981849E12,"submitTime":1.456862981768E12,"finishTime":1.456862986294E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"joseph","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2c317b70-a10a-4bdc-9514-3695a1183639"},{"version":"CommandV1","origId":106394,"guid":"8ed67b54-d747-4fca-af29-de78033ebc16","subtype":"command","commandType":"auto","position":41.25,"command":"%md\n\nThere is a lot more... dig into the docs to find out about belief propogation algorithm now!","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"4e6430c2-19ef-4aaa-a120-2735069057de"},{"version":"CommandV1","origId":106395,"guid":"69dabc79-f5bf-4fcf-9f4c-8e3cf5896435","subtype":"command","commandType":"auto","position":42.25,"command":"displayHTML(frameIt(\"http://graphframes.github.io/user-guide.html#message-passing-via-aggregatemessages\",700))","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<iframe \n src=\"http://graphframes.github.io/user-guide.html#message-passing-via-aggregatemessages\"\n width=\"95%\" height=\"700\"\n sandbox>\n <p>\n <a href=\"http://spark.apache.org/docs/latest/ml-features.html\">\n Fallback link for browsers that, unlikely, don't support frames\n </a>\n </p>\n</iframe>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.46241909104E12,"submitTime":1.462418921085E12,"finishTime":1.462419091149E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":true,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"61f1de37-4ea0-455e-85e1-594f4c031d38"},{"version":"CommandV1","origId":106396,"guid":"6ae5957c-e7bb-46a0-92d4-cf312878a171","subtype":"command","commandType":"auto","position":43.25,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n### prepared by [Raazesh Sainudiin](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [Sivanand Sivaram](https://www.linkedin.com/in/sivanand)\n\n*supported by* [](https://databricks.com/)\nand \n[](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"9a2f94f8-855e-4bc0-8715-7ca77dcaa0ff"}],"dashboards":[],"guid":"dc0d18f4-ce80-47ab-bd0d-97acb40e4095","globalVars":{},"iPythonMetadata":null,"inputWidgets":{}};</script> <script src="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js" onerror="window.mainJsLoadError = true;"></script> </head> <body> <script> if (window.mainJsLoadError) { var u = 'https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js'; var b = document.getElementsByTagName('body')[0]; var c = document.createElement('div'); c.innerHTML = ('<h1>Network Error</h1>' + '<p><b>Please check your network connection and try again.</b></p>' + '<p>Could not load a required resource: ' + u + '</p>'); c.style.margin = '30px'; c.style.padding = '20px 50px'; c.style.backgroundColor = '#f5f5f5'; c.style.borderRadius = '5px'; b.appendChild(c); } </script> </body> </html>