<!DOCTYPE html> <html> <head> <meta name="databricks-html-version" content="1"> <title>scraperUSStateofUnionAddresses - Databricks</title> <meta charset="utf-8"> <meta name="google" content="notranslate"> <meta http-equiv="Content-Language" content="en"> <meta http-equiv="Content-Type" content="text/html; charset=UTF8"> <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Code+Pro:400,700"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/css/bootstrap.min.css"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/jquery-ui-bundle/jquery-ui.min.css"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/main.css"> <link rel="stylesheet" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/print.css" media="print"> <link rel="icon" type="image/png" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/img/favicon.ico"/> <script>window.settings = {"sparkDocsSearchGoogleCx":"004588677886978090460:_rj0wilqwdm","dbcForumURL":"http://forums.databricks.com/","dbfsS3Host":"https://databricks-prod-storage-sydney.s3.amazonaws.com","enableThirdPartyApplicationsUI":false,"enableClusterAcls":false,"notebookRevisionVisibilityHorizon":0,"enableTableHandler":true,"isAdmin":true,"enableLargeResultDownload":false,"nameAndEmail":"Raazesh Sainudiin (r.sainudiin@math.canterbury.ac.nz)","enablePresentationTimerConfig":true,"enableFullTextSearch":true,"enableElasticSparkUI":true,"clusters":true,"hideOffHeapCache":false,"applications":false,"useStaticGuide":false,"fileStoreBase":"FileStore","configurableSparkOptionsSpec":[{"keyPattern":"spark\\.kryo(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.kryo.*","valuePatternDisplay":"*","description":"Configuration options for Kryo serialization"},{"keyPattern":"spark\\.io\\.compression\\.codec","valuePattern":"(lzf|snappy|org\\.apache\\.spark\\.io\\.LZFCompressionCodec|org\\.apache\\.spark\\.io\\.SnappyCompressionCodec)","keyPatternDisplay":"spark.io.compression.codec","valuePatternDisplay":"snappy|lzf","description":"The codec used to compress internal data such as RDD partitions, broadcast variables and shuffle outputs."},{"keyPattern":"spark\\.serializer","valuePattern":"(org\\.apache\\.spark\\.serializer\\.JavaSerializer|org\\.apache\\.spark\\.serializer\\.KryoSerializer)","keyPatternDisplay":"spark.serializer","valuePatternDisplay":"org.apache.spark.serializer.JavaSerializer|org.apache.spark.serializer.KryoSerializer","description":"Class to use for serializing objects that will be sent over the network or need to be cached in serialized form."},{"keyPattern":"spark\\.rdd\\.compress","valuePattern":"(true|false)","keyPatternDisplay":"spark.rdd.compress","valuePatternDisplay":"true|false","description":"Whether to compress serialized RDD partitions (e.g. for StorageLevel.MEMORY_ONLY_SER). Can save substantial space at the cost of some extra CPU time."},{"keyPattern":"spark\\.speculation","valuePattern":"(true|false)","keyPatternDisplay":"spark.speculation","valuePatternDisplay":"true|false","description":"Whether to use speculation (recommended off for streaming)"},{"keyPattern":"spark\\.es(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"es(\\.([^\\.]+))+","valuePattern":".*","keyPatternDisplay":"es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"spark\\.(storage|shuffle)\\.memoryFraction","valuePattern":"0?\\.0*([1-9])([0-9])*","keyPatternDisplay":"spark.(storage|shuffle).memoryFraction","valuePatternDisplay":"(0.0,1.0)","description":"Fraction of Java heap to use for Spark's shuffle or storage"},{"keyPattern":"spark\\.streaming\\.backpressure\\.enabled","valuePattern":"(true|false)","keyPatternDisplay":"spark.streaming.backpressure.enabled","valuePatternDisplay":"true|false","description":"Enables or disables Spark Streaming's internal backpressure mechanism (since 1.5). This enables the Spark Streaming to control the receiving rate based on the current batch scheduling delays and processing times so that the system receives only as fast as the system can process. Internally, this dynamically sets the maximum receiving rate of receivers. This rate is upper bounded by the values `spark.streaming.receiver.maxRate` and `spark.streaming.kafka.maxRatePerPartition` if they are set."},{"keyPattern":"spark\\.streaming\\.receiver\\.maxRate","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.receiver.maxRate","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which each receiver will receive data. Effectively, each stream will consume at most this number of records per second. Setting this configuration to 0 or a negative number will put no limit on the rate. See the deployment guide in the Spark Streaming programing guide for mode details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRatePerPartition","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRatePerPartition","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which data will be read from each Kafka partition when using the Kafka direct stream API introduced in Spark 1.3. See the Kafka Integration guide for more details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRetries","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRetries","valuePatternDisplay":"numeric","description":"Maximum number of consecutive retries the driver will make in order to find the latest offsets on the leader of each partition (a default value of 1 means that the driver will make a maximum of 2 attempts). Only applies to the Kafka direct stream API introduced in Spark 1.3."},{"keyPattern":"spark\\.streaming\\.ui\\.retainedBatches","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.ui.retainedBatches","valuePatternDisplay":"numeric","description":"How many batches the Spark Streaming UI and status APIs remember before garbage collecting."}],"enableReactNotebookComments":true,"enableResetPassword":true,"enableJobsSparkUpgrade":true,"sparkVersions":[{"key":"1.3.x-ubuntu15.10","displayName":"Spark 1.3.0","packageLabel":"spark-1.3-jenkins-ip-10-30-9-162-U0c2673ac85-Sa2ee4664b2-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.4.x-ubuntu15.10","displayName":"Spark 1.4.1","packageLabel":"spark-1.4-jenkins-ip-10-30-9-162-U0c2673ac85-S33a1e4b9c6-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.6.x-ubuntu15.10","displayName":"Spark 1.6.0","packageLabel":"spark-1.6-jenkins-ip-10-30-9-162-U0c2673ac85-Scabba801f3-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"master","displayName":"Spark master (dev)","packageLabel":"","upgradable":true,"deprecated":false,"customerVisible":false}],"enableRestrictedClusterCreation":false,"enableFeedback":false,"defaultNumWorkers":8,"serverContinuationTimeoutMillis":10000,"driverStderrFilePrefix":"stderr","driverStdoutFilePrefix":"stdout","enableSparkDocsSearch":true,"prefetchSidebarNodes":true,"sparkHistoryServerEnabled":true,"sanitizeMarkdownHtml":true,"enableIPythonImportExport":true,"enableNotebookHistoryDiffing":true,"branch":"2.12.3","accountsLimit":-1,"enableNotebookGitBranching":true,"local":false,"displayDefaultContainerMemoryGB":6,"deploymentMode":"production","useSpotForWorkers":false,"enableUserInviteWorkflow":false,"enableStaticNotebooks":true,"dbcGuideURL":"#workspace/databricks_guide/00 Welcome to Databricks","enableCssTransitions":true,"pricingURL":"https://databricks.com/product/pricing","enableClusterAclsConfig":false,"orgId":0,"enableNotebookGitVersioning":true,"files":"files/","enableDriverLogsUI":true,"disableLegacyDashboards":false,"enableWorkspaceAclsConfig":true,"dropzoneMaxFileSize":4096,"enableNewDashboardViews":false,"driverLog4jFilePrefix":"log4j","enableMavenLibraries":true,"displayRowLimit":1000,"defaultSparkVersion":{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},"clusterPublisherRootId":5,"enableLatestJobRunResultPermalink":true,"disallowAddingAdmins":false,"enableSparkConfUI":true,"enableOrgSwitcherUI":false,"clustersLimit":-1,"enableJdbcImport":true,"logfiles":"logfiles/","enableWebappSharding":false,"enableClusterDeltaUpdates":true,"csrfToken":"1f2013f6-c2fd-4ab5-b68c-a2ff4e325639","useFixedStaticNotebookVersionForDevelopment":false,"enableBasicReactDialogBoxes":true,"requireEmailUserName":true,"enableDashboardViews":false,"dbcFeedbackURL":"http://feedback.databricks.com/forums/263785-product-feedback","enableWorkspaceAclService":true,"someName":"Raazesh Sainudiin","enableWorkspaceAcls":true,"gitHash":"0c2673ac858e227cad536fdb45d140aeded238db","userFullname":"Raazesh Sainudiin","enableClusterCreatePage":false,"enableImportFromUrl":true,"enableMiniClusters":false,"enableWebSocketDeltaUpdates":true,"enableDebugUI":false,"showHiddenSparkVersions":false,"allowNonAdminUsers":true,"userId":100005,"dbcSupportURL":"","staticNotebookResourceUrl":"https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/","enableSparkPackages":true,"enableHybridClusterType":false,"enableNotebookHistoryUI":true,"availableWorkspaces":[{"name":"Workspace 0","orgId":0}],"enableFolderHtmlExport":true,"enableSparkVersionsUI":true,"databricksGuideStaticUrl":"","enableHybridClusters":true,"notebookLoadingBackground":"#fff","enableNewJobRunDetailsPage":true,"enableDashboardExport":true,"user":"r.sainudiin@math.canterbury.ac.nz","enableServerAutoComplete":true,"enableStaticHtmlImport":true,"defaultMemoryPerContainerMB":6000,"enablePresenceUI":true,"tablesPublisherRootId":7,"enableNewInputWidgetUI":false,"accounts":true,"enableNewProgressReportUI":true,"defaultCoresPerContainer":4};</script> <script>var __DATABRICKS_NOTEBOOK_MODEL = {"version":"NotebookV1","origId":6442,"name":"scraperUSStateofUnionAddresses","language":"scala","commands":[{"version":"CommandV1","origId":130316,"guid":"8d04ef71-88c1-47ea-a9b2-1e529fa58a9e","subtype":"command","commandType":"auto","position":0.5,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n### prepared by [Raazesh Sainudiin](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [Sivanand Sivaram](https://www.linkedin.com/in/sivanand)\n\n*supported by* [](https://databricks.com/)\nand \n[](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7e0f1e7f-1cc2-4804-9129-31ee0ba588c0"},{"version":"CommandV1","origId":130317,"guid":"ff95526e-f784-4901-ba3d-aa9ef4037b2b","subtype":"command","commandType":"auto","position":0.75,"command":"%md\nThe [html source url](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/db/xtraResources/sdsDatasets/scraperUSStateofUnionAddresses.html) of this databricks notebook and its recorded Uji in context :\n\n[](https://www.youtube.com/v/zgkvusQdNLY?rel=0&autoplay=1&modestbranding=1&start=4613&end=5077)","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"86574401-0b0a-4abd-b9bb-0b4c00dbf02b"},{"version":"CommandV1","origId":6444,"guid":"8304a3b3-6adc-434f-ade5-aa3c0cc16413","subtype":"command","commandType":"auto","position":1.0,"command":"%md\n# Extract, Transform and Load (ETL) of the SoU Addresses\n\n### A bit of bash and lynx to achieve the scraping of the state of the union addresses of the US Presidents\n#### by Paul Brouwers \n### And some Shell-level parsed-data exploration, injection into the distributed file system and testing\n#### by Raazesh Sainudiin\n\nThis SoU dataset is used in the following notebooks:\n* [006_WordCount](/#workspace/scalable-data-science/week2/03_WordCount/006_WordCount).\n\nThe code below is mainly there to show how the text content of each state of the union address was scraped from the following URL:\n* [http://stateoftheunion.onetwothree.net/texts/index.html](http://stateoftheunion.onetwothree.net/texts/index.html)\n\nSuch data acquisition task or ETL is usually the first and crucial step in a data scientist's workflow.\nA data scientist generally does the scraping and parsing of the data by her/himself. \nData ingestion not only allows the scientist to start the analysis but also determines the quality of the analysis by the limits it imposes on the accessible feature space.\n\nWe have done this and put the data in the distributed file system for easy loading into our notebooks for further analysis. This keeps us from having to install unix programs like ``lynx``, ``sed``, etc. that are needed in the shell script below.\n\n```%sh\nfor i in $(lynx --dump http://stateoftheunion.onetwothree.net/texts/index.html | grep texts | grep -v index | sed 's/.*http/http/') ; do lynx --dump $i | tail -n+13 | head -n-14 | sed 's/^\\s\\+//' | sed -e ':a;N;$!ba;s/\\(.\\)\\n/\\1 /g' -e 's/\\n/\\n\\n/' > $(echo $i | sed 's/.*\\([0-9]\\{8\\}\\).*/\\1/').txt ; done\n```\n\nOr in a more atomic form:\n\n```%sh\nfor i in $(lynx --dump http://stateoftheunion.onetwothree.net/texts/index.html \\\n\n | grep texts \\\n\n | grep -v index \\\n\n | sed 's/.*http/http/')\n\ndo \n\n lynx --dump $i \\\n\n | tail -n+13 \\\n\n | head -n-14 \\\n\n | sed 's/^\\s\\+//' \\\n\n | sed -e ':a;N;$!ba;s/\\(.\\)\\n/\\1 /g' -e 's/\\n/\\n\\n/' \\\n\n > $(echo $i | sed 's/.*\\([0-9]\\{8\\}\\).*/\\1/').txt\n\ndone\n```\n**Don't re-evaluate!**\n\nThe following BASH (shell) script can be made to work on databricks cloud directly by installing the dependencies such as ``lynx``, etc. Since we have already scraped it and put the data in our distributed file system **let's not evaluate or ``<Ctrl+Enter>`` the cell below**. The cell is mainly there to show how it can be done (you may want to modify it to scrape other sites for other text data).","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d036f8c1-b513-4515-8f8a-3f08a8666446"},{"version":"CommandV1","origId":6445,"guid":"fb7f9a4f-ee48-490f-8f85-bf577a344693","subtype":"command","commandType":"auto","position":2.5,"command":"%sh\n#remove the hash character from the line below to evaluate when needed\n#for i in $(lynx --dump http://stateoftheunion.onetwothree.net/texts/index.html | grep texts | grep -v index | sed 's/.*http/http/') ; do lynx --dump $i | tail -n+13 | head -n-14 | sed 's/^\\s\\+//' | sed -e ':a;N;$!ba;s/\\(.\\)\\n/\\1 /g' -e 's/\\n/\\n\\n/' > $(echo $i | sed 's/.*\\([0-9]\\{8\\}\\).*/\\1/').txt ; done","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"aff76e75-a6ed-471c-a226-f8835c9d238a"},{"version":"CommandV1","origId":6448,"guid":"9bc9bb88-128e-46b8-9a98-fce0802222bd","subtype":"command","commandType":"auto","position":4.0,"command":"%sh\npwd && ls && du -sh .","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">/databricks/driver\nderby.log\neventlogs\nlogs\n316K\t.\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455835191732E12,"submitTime":1.455835190557E12,"finishTime":1.455835191783E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"291d0a41-1472-412b-8b3c-53f4725a057a"},{"version":"CommandV1","origId":6450,"guid":"cfa7f219-54c4-4da1-9345-6f153f139c7a","subtype":"command","commandType":"auto","position":4.5,"command":"%sh ls /home/ubuntu && du -sh /home/ubuntu","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">databricks\n28K\t/home/ubuntu\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Cancelled","error":null,"startTime":1.455835784654E12,"submitTime":1.455835783514E12,"finishTime":1.455835784741E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"97ee884f-d7f3-472e-80d1-775684eb0251"},{"version":"CommandV1","origId":6449,"guid":"58be0e1c-99fc-4101-9684-d605fac4c505","subtype":"command","commandType":"auto","position":5.0,"command":"%md\nWe can just grab the data as a tarball (gnuZipped tar archive) file ``sou.tar.gz`` using wget as follows:\n\n%sh\nwget http://www.math.canterbury.ac.nz/~r.sainudiin/datasets/public/SOU/sou.tar.gz\n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f51bad32-7236-48e9-9ad4-43cbaf617be7"},{"version":"CommandV1","origId":6491,"guid":"506fef64-f542-4715-a944-e70851579f46","subtype":"command","commandType":"auto","position":6.0,"command":"%sh\ndf -h\npwd","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">Filesystem Size Used Avail Use% Mounted on\n/var/lib/lxc/d984c3c2b0ef8e179895fbe5e1d84779a3f7b1ed/rootfs 296G 12G 272G 5% /\n/dev/xvdb 148G 60M 140G 1% /mnt/ramdisk\nnone 4.0K 0 4.0K 0% /sys/fs/cgroup\nnone 6.1G 116K 6.1G 1% /run\ncgmfs 100K 0 100K 0% /run/cgmanager/fs\nnone 5.0M 0 5.0M 0% /run/lock\nnone 31G 0 31G 0% /run/shm\nnone 100M 0 100M 0% /run/user\ntmpfs 6.1G 0 6.1G 0% /run/user/1000\n/databricks/driver\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.45584081069E12,"submitTime":1.455840809565E12,"finishTime":1.455840810717E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"6ad0b2a4-0aa0-490c-b21b-658348e95009"},{"version":"CommandV1","origId":6492,"guid":"5639befc-0d5f-4cea-b75d-060d3e957bf8","subtype":"command","commandType":"auto","position":7.0,"command":"%sh\nwget http://www.math.canterbury.ac.nz/~r.sainudiin/datasets/public/SOU/sou.tar.gz","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">--2016-02-19 00:13:52-- http://www.math.canterbury.ac.nz/~r.sainudiin/datasets/public/SOU/sou.tar.gz\nResolving www.math.canterbury.ac.nz (www.math.canterbury.ac.nz)... 132.181.26.35\nConnecting to www.math.canterbury.ac.nz (www.math.canterbury.ac.nz)|132.181.26.35|:80... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 3652403 (3.5M) [application/x-gzip]\nSaving to: 'sou.tar.gz'\n\n 0K .......... .......... .......... .......... .......... 1% 316K 11s\n 50K .......... .......... .......... .......... .......... 2% 1.24M 7s\n 100K .......... .......... .......... .......... .......... 4% 1.24M 5s\n 150K .......... .......... .......... .......... .......... 5% 91.8M 4s\n 200K .......... .......... .......... .......... .......... 7% 1.25M 4s\n 250K .......... .......... .......... .......... .......... 8% 68.9M 3s\n 300K .......... .......... .......... .......... .......... 9% 207M 3s\n 350K .......... .......... .......... .......... .......... 11% 1.26M 3s\n 400K .......... .......... .......... .......... .......... 12% 59.8M 2s\n 450K .......... .......... .......... .......... .......... 14% 174M 2s\n 500K .......... .......... .......... .......... .......... 15% 1.26M 2s\n 550K .......... .......... .......... .......... .......... 16% 59.9M 2s\n 600K .......... .......... .......... .......... .......... 18% 124M 2s\n 650K .......... .......... .......... .......... .......... 19% 1.28M 2s\n 700K .......... .......... .......... .......... .......... 21% 73.8M 1s\n 750K .......... .......... .......... .......... .......... 22% 73.0M 1s\n 800K .......... .......... .......... .......... .......... 23% 143M 1s\n 850K .......... .......... .......... .......... .......... 25% 1.27M 1s\n 900K .......... .......... .......... .......... .......... 26% 96.4M 1s\n 950K .......... .......... .......... .......... .......... 28% 78.6M 1s\n 1000K .......... .......... .......... .......... .......... 29% 115M 1s\n 1050K .......... .......... .......... .......... .......... 30% 1.28M 1s\n 1100K .......... .......... .......... .......... .......... 32% 103M 1s\n 1150K .......... .......... .......... .......... .......... 33% 86.8M 1s\n 1200K .......... .......... .......... .......... .......... 35% 101M 1s\n 1250K .......... .......... .......... .......... .......... 36% 1.28M 1s\n 1300K .......... .......... .......... .......... .......... 37% 104M 1s\n 1350K .......... .......... .......... .......... .......... 39% 83.3M 1s\n 1400K .......... .......... .......... .......... .......... 40% 101M 1s\n 1450K .......... .......... .......... .......... .......... 42% 129M 1s\n 1500K .......... .......... .......... .......... .......... 43% 1.29M 1s\n 1550K .......... .......... .......... .......... .......... 44% 86.2M 1s\n 1600K .......... .......... .......... .......... .......... 46% 106M 1s\n 1650K .......... .......... .......... .......... .......... 47% 85.0M 1s\n 1700K .......... .......... .......... .......... .......... 49% 96.9M 1s\n 1750K .......... .......... .......... .......... .......... 50% 1.30M 1s\n 1800K .......... .......... .......... .......... .......... 51% 120M 1s\n 1850K .......... .......... .......... .......... .......... 53% 99.5M 1s\n 1900K .......... .......... .......... .......... .......... 54% 119M 0s\n 1950K .......... .......... .......... .......... .......... 56% 82.0M 0s\n 2000K .......... .......... .......... .......... .......... 57% 1.30M 0s\n 2050K .......... .......... .......... .......... .......... 58% 93.0M 0s\n 2100K .......... .......... .......... .......... .......... 60% 80.5M 0s\n 2150K .......... .......... .......... .......... .......... 61% 99.6M 0s\n 2200K .......... .......... .......... .......... .......... 63% 124M 0s\n 2250K .......... .......... .......... .......... .......... 64% 1.31M 0s\n 2300K .......... .......... .......... .......... .......... 65% 64.9M 0s\n 2350K .......... .......... .......... .......... .......... 67% 76.7M 0s\n 2400K .......... .......... .......... .......... .......... 68% 95.4M 0s\n 2450K .......... .......... .......... .......... .......... 70% 82.0M 0s\n 2500K .......... .......... .......... .......... .......... 71% 227M 0s\n 2550K .......... .......... .......... .......... .......... 72% 248M 0s\n 2600K .......... .......... .......... .......... .......... 74% 1.30M 0s\n 2650K .......... .......... .......... .......... .......... 75% 124M 0s\n 2700K .......... .......... .......... .......... .......... 77% 55.5M 0s\n 2750K .......... .......... .......... .......... .......... 78% 98.5M 0s\n 2800K .......... .......... .......... .......... .......... 79% 106M 0s\n 2850K .......... .......... .......... .......... .......... 81% 195M 0s\n 2900K .......... .......... .......... .......... .......... 82% 1.32M 0s\n 2950K .......... .......... .......... .......... .......... 84% 64.1M 0s\n 3000K .......... .......... .......... .......... .......... 85% 101M 0s\n 3050K .......... .......... .......... .......... .......... 86% 71.2M 0s\n 3100K .......... .......... .......... .......... .......... 88% 98.7M 0s\n 3150K .......... .......... .......... .......... .......... 89% 209M 0s\n 3200K .......... .......... .......... .......... .......... 91% 1.33M 0s\n 3250K .......... .......... .......... .......... .......... 92% 99.6M 0s\n 3300K .......... .......... .......... .......... .......... 93% 105M 0s\n 3350K .......... .......... .......... .......... .......... 95% 77.9M 0s\n 3400K .......... .......... .......... .......... .......... 96% 123M 0s\n 3450K .......... .......... .......... .......... .......... 98% 77.8M 0s\n 3500K .......... .......... .......... .......... .......... 99% 166M 0s\n 3550K .......... ...... 100% 230M=0.8s\n\n2016-02-19 00:13:53 (4.38 MB/s) - 'sou.tar.gz' saved [3652403/3652403]\n\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455840832594E12,"submitTime":1.45584083149E12,"finishTime":1.455840833611E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d224d0b8-c49b-4d5a-b0b6-22d258c18505"},{"version":"CommandV1","origId":6493,"guid":"6db1d7c7-d15f-4333-9419-ccdcd3464d4c","subtype":"command","commandType":"auto","position":8.0,"command":"%sh\nls","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">derby.log\neventlogs\nlogs\nsou.tar.gz\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455843206932E12,"submitTime":1.455843205837E12,"finishTime":1.455843206983E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"09fa3f31-2acd-47ac-8c87-6c7a255a1796"},{"version":"CommandV1","origId":6494,"guid":"cdfc75dc-d129-4446-8b5e-39930153cdb2","subtype":"command","commandType":"auto","position":9.0,"command":"%sh\nenv","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">MASTER=spark://10.135.239.66:7077\nSPARK_HOME=/databricks/spark\nSHELL=/bin/bash\nTERM=unknown\nSCALA_VERSION=2.10\nCLUSTER_DB_HOME=/databricks\nDB_HOME=/databricks\nSPARK_LOCAL_DIRS=/local_disk0\nUSER=root\nJAVA_OPTS= -XX:MaxPermSize=512m -XX:-OmitStackTraceInFastThrow -Xms3776m -Xmx3776m -Djava.library.path= -Dspark.ui.port=43140 -javaagent:/databricks/DatabricksAgent.jar -XX:+PrintFlagsFinal -XX:+PrintGCDateStamps -verbose:gc -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -Dspark.executor.extraJavaOptions="-XX:ReservedCodeCacheSize=256m -XX:+UseCodeCacheFlushing -javaagent:/databricks/DatabricksAgent.jar -XX:+PrintFlagsFinal -XX:+PrintGCDateStamps -verbose:gc -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -Ddatabricks.serviceName=spark-executor-1" -Dspark.executor.memory=4800m -Dspark.executor.extraClassPath=/databricks/spark/dbconf/log4j/executor:/databricks/spark/dbconf/jets3t/:/databricks/spark/dbconf/hadoop:/databricks/hive/conf:/databricks/jars/FastInfoset-1.2.12.jar:/databricks/jars/HikariCP-2.4.1.jar:/databricks/jars/JavaEWAH-0.3.2.jar:/databricks/jars/REngine-2.1.0.jar:/databricks/jars/RoaringBitmap-0.5.11.jar:/databricks/jars/Rserve-1.8-3.jar:/databricks/jars/ST4-4.0.4.jar:/databricks/jars/akka-actor_2.10-2.3.4-spark.jar:/databricks/jars/akka-remote_2.10-2.3.4-spark.jar:/databricks/jars/akka-slf4j_2.10-2.3.4-spark.jar:/databricks/jars/ant-1.9.2.jar:/databricks/jars/ant-jsch-1.9.2.jar:/databricks/jars/ant-launcher-1.9.2.jar:/databricks/jars/antlr-2.7.7.jar:/databricks/jars/antlr-runtime-3.4.jar:/databricks/jars/apache-log4j-extras-1.2.17.jar:/databricks/jars/api-base_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/arpack_combined_all-0.1.jar:/databricks/jars/asm-3.1.jar:/databricks/jars/avro-1.7.7.jar:/databricks/jars/avro-ipc-1.7.7-tests.jar:/databricks/jars/avro-ipc-1.7.7.jar:/databricks/jars/avro-mapred-1.7.7-hadoop1.jar:/databricks/jars/aws-java-sdk-1.9.40.jar:/databricks/jars/aws-java-sdk-autoscaling-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudformation-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudfront-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudhsm-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudsearch-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudtrail-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudwatch-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudwatchmetrics-1.9.40.jar:/databricks/jars/aws-java-sdk-codedeploy-1.9.40.jar:/databricks/jars/aws-java-sdk-cognitoidentity-1.9.40.jar:/databricks/jars/aws-java-sdk-cognitosync-1.9.40.jar:/databricks/jars/aws-java-sdk-config-1.9.40.jar:/databricks/jars/aws-java-sdk-core-1.9.40.jar:/databricks/jars/aws-java-sdk-datapipeline-1.9.40.jar:/databricks/jars/aws-java-sdk-directconnect-1.9.40.jar:/databricks/jars/aws-java-sdk-directory-1.9.40.jar:/databricks/jars/aws-java-sdk-dynamodb-1.9.40.jar:/databricks/jars/aws-java-sdk-ec2-1.9.40.jar:/databricks/jars/aws-java-sdk-ecs-1.9.40.jar:/databricks/jars/aws-java-sdk-efs-1.9.40.jar:/databricks/jars/aws-java-sdk-elasticache-1.9.40.jar:/databricks/jars/aws-java-sdk-elasticbeanstalk-1.9.40.jar:/databricks/jars/aws-java-sdk-elasticloadbalancing-1.9.40.jar:/databricks/jars/aws-java-sdk-elastictranscoder-1.9.40.jar:/databricks/jars/aws-java-sdk-emr-1.9.40.jar:/databricks/jars/aws-java-sdk-glacier-1.9.40.jar:/databricks/jars/aws-java-sdk-iam-1.9.40.jar:/databricks/jars/aws-java-sdk-importexport-1.9.40.jar:/databricks/jars/aws-java-sdk-kinesis-1.9.40.jar:/databricks/jars/aws-java-sdk-kms-1.9.40.jar:/databricks/jars/aws-java-sdk-lambda-1.9.40.jar:/databricks/jars/aws-java-sdk-logs-1.9.40.jar:/databricks/jars/aws-java-sdk-machinelearning-1.9.40.jar:/databricks/jars/aws-java-sdk-opsworks-1.9.40.jar:/databricks/jars/aws-java-sdk-rds-1.9.40.jar:/databricks/jars/aws-java-sdk-redshift-1.9.40.jar:/databricks/jars/aws-java-sdk-route53-1.9.40.jar:/databricks/jars/aws-java-sdk-s3-1.9.40.jar:/databricks/jars/aws-java-sdk-ses-1.9.40.jar:/databricks/jars/aws-java-sdk-simpledb-1.9.40.jar:/databricks/jars/aws-java-sdk-simpleworkflow-1.9.40.jar:/databricks/jars/aws-java-sdk-sns-1.9.40.jar:/databricks/jars/aws-java-sdk-sqs-1.9.40.jar:/databricks/jars/aws-java-sdk-ssm-1.9.40.jar:/databricks/jars/aws-java-sdk-storagegateway-1.9.40.jar:/databricks/jars/aws-java-sdk-sts-1.9.40.jar:/databricks/jars/aws-java-sdk-support-1.9.40.jar:/databricks/jars/aws-java-sdk-swf-libraries-1.9.40.jar:/databricks/jars/aws-java-sdk-workspaces-1.9.40.jar:/databricks/jars/bonecp-0.8.0.RELEASE.jar:/databricks/jars/breeze-macros_2.10-0.11.2.jar:/databricks/jars/breeze_2.10-0.11.2.jar:/databricks/jars/c3p0-0.9.5.1.jar:/databricks/jars/calcite-avatica-1.2.0-incubating.jar:/databricks/jars/calcite-core-1.2.0-incubating.jar:/databricks/jars/calcite-linq4j-1.2.0-incubating.jar:/databricks/jars/chauffeur-api_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/chill-java-0.5.0.jar:/databricks/jars/chill_2.10-0.5.0.jar:/databricks/jars/classmate-1.0.0.jar:/databricks/jars/com.sun.el-1.0.0.v201105211818.jar:/databricks/jars/common_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/commons-beanutils-1.7.0.jar:/databricks/jars/commons-beanutils-core-1.8.0.jar:/databricks/jars/commons-cli-1.2.jar:/databricks/jars/commons-codec-1.10.jar:/databricks/jars/commons-collections-3.2.1.jar:/databricks/jars/commons-compiler-2.7.8.jar:/databricks/jars/commons-compress-1.4.1.jar:/databricks/jars/commons-configuration-1.6.jar:/databricks/jars/commons-csv-1.1.jar:/databricks/jars/commons-dbcp-1.4.jar:/databricks/jars/commons-digester-1.8.jar:/databricks/jars/commons-el-1.0.jar:/databricks/jars/commons-fileupload-1.3.jar:/databricks/jars/commons-httpclient-3.1.jar:/databricks/jars/commons-io-2.4.jar:/databricks/jars/commons-lang-2.6.jar:/databricks/jars/commons-lang3-3.3.2.jar:/databricks/jars/commons-logging-1.2.jar:/databricks/jars/commons-math-2.1.jar:/databricks/jars/commons-math3-3.4.1.jar:/databricks/jars/commons-net-2.2.jar:/databricks/jars/commons-pool-1.5.4.jar:/databricks/jars/compilerplugin_2.10-0.4.15-9.jar:/databricks/jars/compress-lzf-1.0.3.jar:/databricks/jars/config-1.2.1.jar:/databricks/jars/core-1.1.2.jar:/databricks/jars/curator-client-2.4.0.jar:/databricks/jars/curator-framework-2.4.0.jar:/databricks/jars/curator-recipes-2.4.0.jar:/databricks/jars/data-client_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/data-common_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/datanucleus-api-jdo-3.2.6.jar:/databricks/jars/datanucleus-core-3.2.10.jar:/databricks/jars/datanucleus-rdbms-3.2.9.jar:/databricks/jars/derby-10.10.2.0.jar:/databricks/jars/dockertools_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/driver-daemon_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/eigenbase-properties-1.1.5.jar:/databricks/jars/extern_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/gmetric4j-1.0.7.jar:/databricks/jars/groovy-all-2.1.6.jar:/databricks/jars/guava-14.0.1.jar:/databricks/jars/h2-1.3.174.jar:/databricks/jars/hadoop-client-1.2.1-0.jar:/databricks/jars/hadoop-core-1.2.1-0.jar:/databricks/jars/hibernate-validator-5.1.1.Final.jar:/databricks/jars/hive-beeline-1.2.1.spark.jar:/databricks/jars/hive-cli-1.2.1.spark.jar:/databricks/jars/hive-exec-1.2.1.spark.jar:/databricks/jars/hive-jdbc-1.2.1.spark.jar:/databricks/jars/hive-metastore-1.2.1.spark.jar:/databricks/jars/hive-service-1.2.1.spark.jar:/databricks/jars/hsqldb-1.8.0.10.jar:/databricks/jars/httpclient-4.4.1.jar:/databricks/jars/httpcore-4.4.1.jar:/databricks/jars/istack-commons-runtime-2.16.jar:/databricks/jars/ivy-2.4.0.jar:/databricks/jars/jackson-annotations-2.4.5.jar:/databricks/jars/jackson-core-2.4.5.jar:/databricks/jars/jackson-core-asl-1.9.13.jar:/databricks/jars/jackson-databind-2.4.5.jar:/databricks/jars/jackson-datatype-joda-2.4.5.jar:/databricks/jars/jackson-jaxrs-1.7.1.jar:/databricks/jars/jackson-mapper-asl-1.9.13.jar:/databricks/jars/jackson-module-scala_2.10-2.4.5.jar:/databricks/jars/jackson-xc-1.7.1.jar:/databricks/jars/janino-2.7.8.jar:/databricks/jars/jansi-1.4.jar:/databricks/jars/javax.el-2.1.0.v201105211819.jar:/databricks/jars/javax.servlet-3.0.0.v201112011016.jar:/databricks/jars/javax.servlet.jsp-2.1.0.v201105211820.jar:/databricks/jars/javax.servlet.jsp.jstl-1.2.0.v201105211821.jar:/databricks/jars/javolution-5.5.1.jar:/databricks/jars/jaxb-api-2.2.7.jar:/databricks/jars/jaxb-core-2.2.7.jar:/databricks/jars/jaxb-impl-2.2.7.jar:/databricks/jars/jboss-logging-3.1.3.GA.jar:/databricks/jars/jcl-over-slf4j-1.7.10.jar:/databricks/jars/jdbc4-1.1.7.1007.jar:/databricks/jars/jdbi-2.63.1.jar:/databricks/jars/jdo-api-3.0.1.jar:/databricks/jars/jersey-core-1.8.jar:/databricks/jars/jersey-json-1.8.jar:/databricks/jars/jersey-server-1.8.jar:/databricks/jars/jets3t-0.7.1-0.jar:/databricks/jars/jets3t-0.7.1.jar:/databricks/jars/jettison-1.1.jar:/databricks/jars/jetty-client-8.1.14.v20131031.jar:/databricks/jars/jetty-continuation-8.1.14.v20131031.jar:/databricks/jars/jetty-http-8.1.14.v20131031.jar:/databricks/jars/jetty-io-8.1.14.v20131031.jar:/databricks/jars/jetty-jsp-7.6.8.v20121106.jar:/databricks/jars/jetty-security-8.1.14.v20131031.jar:/databricks/jars/jetty-server-8.1.14.v20131031.jar:/databricks/jars/jetty-servlet-8.1.14.v20131031.jar:/databricks/jars/jetty-servlets-8.1.14.v20131031.jar:/databricks/jars/jetty-util-8.1.14.v20131031.jar:/databricks/jars/jetty-webapp-7.6.8.v20121106.jar:/databricks/jars/jetty-xml-7.6.8.v20121106.jar:/databricks/jars/jetty8-client_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/jline-2.12.jar:/databricks/jars/jline.jar:/databricks/jars/jniloader-1.1.jar:/databricks/jars/joda-convert-1.6.jar:/databricks/jars/joda-time-2.9.jar:/databricks/jars/jodd-core-3.5.2.jar:/databricks/jars/jpam-1.1.jar:/databricks/jars/jsch-0.1.50.jar:/databricks/jars/json-20090211.jar:/databricks/jars/json4s-ast_2.10-3.2.10.jar:/databricks/jars/json4s-core_2.10-3.2.10.jar:/databricks/jars/json4s-jackson_2.10-3.2.10.jar:/databricks/jars/jsonutil_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/jsr173_api-1.0.jar:/databricks/jars/jsr305-2.0.1.jar:/databricks/jars/jta-1.1.jar:/databricks/jars/jtransforms-2.4.0.jar:/databricks/jars/jul-to-slf4j-1.7.10.jar:/databricks/jars/kryo-2.21.jar:/databricks/jars/lenses_2.10-0.3.jar:/databricks/jars/leveldbjni-all-1.8.jar:/databricks/jars/libfb303-0.9.2.jar:/databricks/jars/libthrift-0.9.2.jar:/databricks/jars/lift-json_2.10-2.5.1.jar:/databricks/jars/log4j-1.2.17.jar:/databricks/jars/lz4-1.3.0.jar:/databricks/jars/mchange-commons-java-0.2.10.jar:/databricks/jars/mesos-0.21.1-shaded-protobuf.jar:/databricks/jars/metrics-core-3.0.2.jar:/databricks/jars/metrics-core-3.1.2.jar:/databricks/jars/metrics-ganglia-3.0.2.jar:/databricks/jars/metrics-ganglia-3.1.2.jar:/databricks/jars/metrics-graphite-3.1.2.jar:/databricks/jars/metrics-healthchecks-3.0.2.jar:/databricks/jars/metrics-jdbi-3.0.2.jar:/databricks/jars/metrics-jetty8-3.0.2.jar:/databricks/jars/metrics-json-3.0.2.jar:/databricks/jars/metrics-json-3.1.2.jar:/databricks/jars/metrics-jvm-3.0.2.jar:/databricks/jars/metrics-jvm-3.1.2.jar:/databricks/jars/metrics-log4j-3.0.2.jar:/databricks/jars/metrics-servlets-3.0.2.jar:/databricks/jars/minlog-1.2.jar:/databricks/jars/mysql-connector-java-5.1.27.jar:/databricks/jars/native_ref-java-1.1.jar:/databricks/jars/native_system-java-1.1.jar:/databricks/jars/netlib-native_ref-linux-armhf-1.1-natives.jar:/databricks/jars/netlib-native_ref-linux-i686-1.1-natives.jar:/databricks/jars/netlib-native_ref-linux-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_ref-osx-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_ref-win-i686-1.1-natives.jar:/databricks/jars/netlib-native_ref-win-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_system-linux-armhf-1.1-natives.jar:/databricks/jars/netlib-native_system-linux-i686-1.1-natives.jar:/databricks/jars/netlib-native_system-linux-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_system-osx-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_system-win-i686-1.1-natives.jar:/databricks/jars/netlib-native_system-win-x86_64-1.1-natives.jar:/databricks/jars/netty-3.8.0.Final.jar:/databricks/jars/netty-all-4.0.29.Final.jar:/databricks/jars/objenesis-1.2.jar:/databricks/jars/oncrpc-1.0.7.jar:/databricks/jars/opencsv-2.3.jar:/databricks/jars/org.apache.jasper.glassfish-2.1.0.v201110031002.jar:/databricks/jars/org.apache.taglibs.standard.glassfish-1.2.0.v201112081803.jar:/databricks/jars/org.eclipse.jdt.core-3.7.1.jar:/databricks/jars/oro-2.0.8.jar:/databricks/jars/paranamer-2.6.jar:/databricks/jars/parquet-column-1.7.0.jar:/databricks/jars/parquet-common-1.7.0.jar:/databricks/jars/parquet-encoding-1.7.0.jar:/databricks/jars/parquet-format-2.3.0-incubating.jar:/databricks/jars/parquet-generator-1.7.0.jar:/databricks/jars/parquet-hadoop-1.7.0.jar:/databricks/jars/parquet-hadoop-bundle-1.6.0.jar:/databricks/jars/parquet-jackson-1.7.0.jar:/databricks/jars/pmml-agent-1.1.15.jar:/databricks/jars/pmml-model-1.1.15.jar:/databricks/jars/pmml-schema-1.1.15.jar:/databricks/jars/postgresql-9.4-1204-jdbc41.jar:/databricks/jars/protobuf-java-2.5.0-spark.jar:/databricks/jars/py4j-0.9.jar:/databricks/jars/pyrolite-4.9.jar:/databricks/jars/quasiquotes_2.10-2.0.0.jar:/databricks/jars/reflectasm-1.07-shaded.jar:/databricks/jars/s3_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/scala-compiler.jar:/databricks/jars/scala-library.jar:/databricks/jars/scala-reflect.jar:/databricks/jars/scalalogging-slf4j_2.10-1.1.0.jar:/databricks/jars/scalap-2.10.5.jar:/databricks/jars/scalapb-runtime_2.10-0.4.15-9.jar:/databricks/jars/slf4j-api-1.7.10.jar:/databricks/jars/slf4j-log4j12-1.7.10.jar:/databricks/jars/snappy-0.2.jar:/databricks/jars/snappy-java-1.1.2.jar:/databricks/jars/spark-avro_2.10-2.0.1.jar:/databricks/jars/spark-bagel_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-catalyst_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-core_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-csv_2.10-1.3.0.jar:/databricks/jars/spark-ganglia-lgpl_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-graphx_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-hive-thriftserver_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-hive_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-launcher_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-mllib_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-network-common_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-network-shuffle_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-redshift_2.10-0.6.0.jar:/databricks/jars/spark-repl_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-sql_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-streaming-twitter_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-streaming_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-unsafe_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spire-macros_2.10-0.7.4.jar:/databricks/jars/spire_2.10-0.7.4.jar:/databricks/jars/spring-core-4.1.4.RELEASE.jar:/databricks/jars/spring-test-4.1.4.RELEASE.jar:/databricks/jars/sqlite-jdbc-3.8.11.2.jar:/databricks/jars/stax-api-1.0.1.jar:/databricks/jars/stream-2.7.0.jar:/databricks/jars/stringtemplate-3.2.1.jar:/databricks/jars/super-csv-2.2.0.jar:/databricks/jars/tachyon-0.5.0-0.jar:/databricks/jars/twitter4j-core-4.0.4.jar:/databricks/jars/twitter4j-stream-4.0.4.jar:/databricks/jars/uncommons-maths-1.2.2a.jar:/databricks/jars/univocity-parsers-1.5.1.jar:/databricks/jars/unused-1.0.0.jar:/databricks/jars/util-app_2.10-6.23.0.jar:/databricks/jars/util-core_2.10-6.23.0.jar:/databricks/jars/util-jvm_2.10-6.23.0.jar:/databricks/jars/validation-api-1.1.0.Final.jar:/databricks/jars/vigdis_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/xbean-asm5-shaded-4.4.jar:/databricks/jars/xmlenc-0.52.jar:/databricks/jars/xz-1.0.jar:/databricks/jars/zookeeper-3.4.5.jar\nSUDO_USER=root\nSUDO_UID=0\nSPARK_PUBLIC_DNS=10.135.239.66\nPYSPARK_PYTHON=/databricks/python/bin/python\nUSERNAME=root\nCOLUMNS=80\nSPARK_WORKER_MEMORY=6000m\nMAIL=/var/mail/root\nPATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n_=/usr/bin/env\nHIVE_HOME=/home/ubuntu/hive-0.9.0-bin\nPWD=/databricks/driver\nJAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/\nLANG=en_US.UTF-8\nDRIVER_PID_FILE=/tmp/driver-daemon.pid\nPYSPARK_GATEWAY_PORT=55640\nLINES=24\nSPARK_ENV_LOADED=1\nSHLVL=4\nHOME=/root\nSUDO_COMMAND=/usr/bin/lxc-attach -n 0218-181130-fence5_10_135_239_66 -- env DB_HOME=/databricks CLUSTER_DB_HOME=/databricks bash -c bash ${DB_HOME:-/home/ubuntu/databricks}/spark/scripts/start_chauffeur.sh /tmp/chauffeur-env.sh\nSPARK_LOCAL_IP=10.135.239.66\nPYTHONPATH=/databricks/spark/python:/databricks/spark/python/lib/py4j-0.8.2.1-src.zip:/databricks/jars/driver-daemon_2.10-1.3.2-SNAPSHOT.jar:/databricks/spark/python\nLOGNAME=root\nCLASSPATH=/databricks/spark/dbconf/jets3t/:/databricks/spark/dbconf/log4j/driver:/databricks/hive/conf:/databricks/spark/dbconf/hadoop:/databricks/jars/FastInfoset-1.2.12.jar:/databricks/jars/HikariCP-2.4.1.jar:/databricks/jars/JavaEWAH-0.3.2.jar:/databricks/jars/REngine-2.1.0.jar:/databricks/jars/RoaringBitmap-0.5.11.jar:/databricks/jars/Rserve-1.8-3.jar:/databricks/jars/ST4-4.0.4.jar:/databricks/jars/akka-actor_2.10-2.3.4-spark.jar:/databricks/jars/akka-remote_2.10-2.3.4-spark.jar:/databricks/jars/akka-slf4j_2.10-2.3.4-spark.jar:/databricks/jars/ant-1.9.2.jar:/databricks/jars/ant-jsch-1.9.2.jar:/databricks/jars/ant-launcher-1.9.2.jar:/databricks/jars/antlr-2.7.7.jar:/databricks/jars/antlr-runtime-3.4.jar:/databricks/jars/apache-log4j-extras-1.2.17.jar:/databricks/jars/api-base_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/arpack_combined_all-0.1.jar:/databricks/jars/asm-3.1.jar:/databricks/jars/avro-1.7.7.jar:/databricks/jars/avro-ipc-1.7.7-tests.jar:/databricks/jars/avro-ipc-1.7.7.jar:/databricks/jars/avro-mapred-1.7.7-hadoop1.jar:/databricks/jars/aws-java-sdk-1.9.40.jar:/databricks/jars/aws-java-sdk-autoscaling-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudformation-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudfront-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudhsm-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudsearch-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudtrail-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudwatch-1.9.40.jar:/databricks/jars/aws-java-sdk-cloudwatchmetrics-1.9.40.jar:/databricks/jars/aws-java-sdk-codedeploy-1.9.40.jar:/databricks/jars/aws-java-sdk-cognitoidentity-1.9.40.jar:/databricks/jars/aws-java-sdk-cognitosync-1.9.40.jar:/databricks/jars/aws-java-sdk-config-1.9.40.jar:/databricks/jars/aws-java-sdk-core-1.9.40.jar:/databricks/jars/aws-java-sdk-datapipeline-1.9.40.jar:/databricks/jars/aws-java-sdk-directconnect-1.9.40.jar:/databricks/jars/aws-java-sdk-directory-1.9.40.jar:/databricks/jars/aws-java-sdk-dynamodb-1.9.40.jar:/databricks/jars/aws-java-sdk-ec2-1.9.40.jar:/databricks/jars/aws-java-sdk-ecs-1.9.40.jar:/databricks/jars/aws-java-sdk-efs-1.9.40.jar:/databricks/jars/aws-java-sdk-elasticache-1.9.40.jar:/databricks/jars/aws-java-sdk-elasticbeanstalk-1.9.40.jar:/databricks/jars/aws-java-sdk-elasticloadbalancing-1.9.40.jar:/databricks/jars/aws-java-sdk-elastictranscoder-1.9.40.jar:/databricks/jars/aws-java-sdk-emr-1.9.40.jar:/databricks/jars/aws-java-sdk-glacier-1.9.40.jar:/databricks/jars/aws-java-sdk-iam-1.9.40.jar:/databricks/jars/aws-java-sdk-importexport-1.9.40.jar:/databricks/jars/aws-java-sdk-kinesis-1.9.40.jar:/databricks/jars/aws-java-sdk-kms-1.9.40.jar:/databricks/jars/aws-java-sdk-lambda-1.9.40.jar:/databricks/jars/aws-java-sdk-logs-1.9.40.jar:/databricks/jars/aws-java-sdk-machinelearning-1.9.40.jar:/databricks/jars/aws-java-sdk-opsworks-1.9.40.jar:/databricks/jars/aws-java-sdk-rds-1.9.40.jar:/databricks/jars/aws-java-sdk-redshift-1.9.40.jar:/databricks/jars/aws-java-sdk-route53-1.9.40.jar:/databricks/jars/aws-java-sdk-s3-1.9.40.jar:/databricks/jars/aws-java-sdk-ses-1.9.40.jar:/databricks/jars/aws-java-sdk-simpledb-1.9.40.jar:/databricks/jars/aws-java-sdk-simpleworkflow-1.9.40.jar:/databricks/jars/aws-java-sdk-sns-1.9.40.jar:/databricks/jars/aws-java-sdk-sqs-1.9.40.jar:/databricks/jars/aws-java-sdk-ssm-1.9.40.jar:/databricks/jars/aws-java-sdk-storagegateway-1.9.40.jar:/databricks/jars/aws-java-sdk-sts-1.9.40.jar:/databricks/jars/aws-java-sdk-support-1.9.40.jar:/databricks/jars/aws-java-sdk-swf-libraries-1.9.40.jar:/databricks/jars/aws-java-sdk-workspaces-1.9.40.jar:/databricks/jars/bonecp-0.8.0.RELEASE.jar:/databricks/jars/breeze-macros_2.10-0.11.2.jar:/databricks/jars/breeze_2.10-0.11.2.jar:/databricks/jars/c3p0-0.9.5.1.jar:/databricks/jars/calcite-avatica-1.2.0-incubating.jar:/databricks/jars/calcite-core-1.2.0-incubating.jar:/databricks/jars/calcite-linq4j-1.2.0-incubating.jar:/databricks/jars/chauffeur-api_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/chill-java-0.5.0.jar:/databricks/jars/chill_2.10-0.5.0.jar:/databricks/jars/classmate-1.0.0.jar:/databricks/jars/com.sun.el-1.0.0.v201105211818.jar:/databricks/jars/common_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/commons-beanutils-1.7.0.jar:/databricks/jars/commons-beanutils-core-1.8.0.jar:/databricks/jars/commons-cli-1.2.jar:/databricks/jars/commons-codec-1.10.jar:/databricks/jars/commons-collections-3.2.1.jar:/databricks/jars/commons-compiler-2.7.8.jar:/databricks/jars/commons-compress-1.4.1.jar:/databricks/jars/commons-configuration-1.6.jar:/databricks/jars/commons-csv-1.1.jar:/databricks/jars/commons-dbcp-1.4.jar:/databricks/jars/commons-digester-1.8.jar:/databricks/jars/commons-el-1.0.jar:/databricks/jars/commons-fileupload-1.3.jar:/databricks/jars/commons-httpclient-3.1.jar:/databricks/jars/commons-io-2.4.jar:/databricks/jars/commons-lang-2.6.jar:/databricks/jars/commons-lang3-3.3.2.jar:/databricks/jars/commons-logging-1.2.jar:/databricks/jars/commons-math-2.1.jar:/databricks/jars/commons-math3-3.4.1.jar:/databricks/jars/commons-net-2.2.jar:/databricks/jars/commons-pool-1.5.4.jar:/databricks/jars/compilerplugin_2.10-0.4.15-9.jar:/databricks/jars/compress-lzf-1.0.3.jar:/databricks/jars/config-1.2.1.jar:/databricks/jars/core-1.1.2.jar:/databricks/jars/curator-client-2.4.0.jar:/databricks/jars/curator-framework-2.4.0.jar:/databricks/jars/curator-recipes-2.4.0.jar:/databricks/jars/data-client_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/data-common_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/datanucleus-api-jdo-3.2.6.jar:/databricks/jars/datanucleus-core-3.2.10.jar:/databricks/jars/datanucleus-rdbms-3.2.9.jar:/databricks/jars/derby-10.10.2.0.jar:/databricks/jars/dockertools_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/driver-daemon_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/eigenbase-properties-1.1.5.jar:/databricks/jars/extern_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/gmetric4j-1.0.7.jar:/databricks/jars/groovy-all-2.1.6.jar:/databricks/jars/guava-14.0.1.jar:/databricks/jars/h2-1.3.174.jar:/databricks/jars/hadoop-client-1.2.1-0.jar:/databricks/jars/hadoop-core-1.2.1-0.jar:/databricks/jars/hibernate-validator-5.1.1.Final.jar:/databricks/jars/hive-beeline-1.2.1.spark.jar:/databricks/jars/hive-cli-1.2.1.spark.jar:/databricks/jars/hive-exec-1.2.1.spark.jar:/databricks/jars/hive-jdbc-1.2.1.spark.jar:/databricks/jars/hive-metastore-1.2.1.spark.jar:/databricks/jars/hive-service-1.2.1.spark.jar:/databricks/jars/hsqldb-1.8.0.10.jar:/databricks/jars/httpclient-4.4.1.jar:/databricks/jars/httpcore-4.4.1.jar:/databricks/jars/istack-commons-runtime-2.16.jar:/databricks/jars/ivy-2.4.0.jar:/databricks/jars/jackson-annotations-2.4.5.jar:/databricks/jars/jackson-core-2.4.5.jar:/databricks/jars/jackson-core-asl-1.9.13.jar:/databricks/jars/jackson-databind-2.4.5.jar:/databricks/jars/jackson-datatype-joda-2.4.5.jar:/databricks/jars/jackson-jaxrs-1.7.1.jar:/databricks/jars/jackson-mapper-asl-1.9.13.jar:/databricks/jars/jackson-module-scala_2.10-2.4.5.jar:/databricks/jars/jackson-xc-1.7.1.jar:/databricks/jars/janino-2.7.8.jar:/databricks/jars/jansi-1.4.jar:/databricks/jars/javax.el-2.1.0.v201105211819.jar:/databricks/jars/javax.servlet-3.0.0.v201112011016.jar:/databricks/jars/javax.servlet.jsp-2.1.0.v201105211820.jar:/databricks/jars/javax.servlet.jsp.jstl-1.2.0.v201105211821.jar:/databricks/jars/javolution-5.5.1.jar:/databricks/jars/jaxb-api-2.2.7.jar:/databricks/jars/jaxb-core-2.2.7.jar:/databricks/jars/jaxb-impl-2.2.7.jar:/databricks/jars/jboss-logging-3.1.3.GA.jar:/databricks/jars/jcl-over-slf4j-1.7.10.jar:/databricks/jars/jdbc4-1.1.7.1007.jar:/databricks/jars/jdbi-2.63.1.jar:/databricks/jars/jdo-api-3.0.1.jar:/databricks/jars/jersey-core-1.8.jar:/databricks/jars/jersey-json-1.8.jar:/databricks/jars/jersey-server-1.8.jar:/databricks/jars/jets3t-0.7.1-0.jar:/databricks/jars/jets3t-0.7.1.jar:/databricks/jars/jettison-1.1.jar:/databricks/jars/jetty-client-8.1.14.v20131031.jar:/databricks/jars/jetty-continuation-8.1.14.v20131031.jar:/databricks/jars/jetty-http-8.1.14.v20131031.jar:/databricks/jars/jetty-io-8.1.14.v20131031.jar:/databricks/jars/jetty-jsp-7.6.8.v20121106.jar:/databricks/jars/jetty-security-8.1.14.v20131031.jar:/databricks/jars/jetty-server-8.1.14.v20131031.jar:/databricks/jars/jetty-servlet-8.1.14.v20131031.jar:/databricks/jars/jetty-servlets-8.1.14.v20131031.jar:/databricks/jars/jetty-util-8.1.14.v20131031.jar:/databricks/jars/jetty-webapp-7.6.8.v20121106.jar:/databricks/jars/jetty-xml-7.6.8.v20121106.jar:/databricks/jars/jetty8-client_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/jline-2.12.jar:/databricks/jars/jline.jar:/databricks/jars/jniloader-1.1.jar:/databricks/jars/joda-convert-1.6.jar:/databricks/jars/joda-time-2.9.jar:/databricks/jars/jodd-core-3.5.2.jar:/databricks/jars/jpam-1.1.jar:/databricks/jars/jsch-0.1.50.jar:/databricks/jars/json-20090211.jar:/databricks/jars/json4s-ast_2.10-3.2.10.jar:/databricks/jars/json4s-core_2.10-3.2.10.jar:/databricks/jars/json4s-jackson_2.10-3.2.10.jar:/databricks/jars/jsonutil_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/jsr173_api-1.0.jar:/databricks/jars/jsr305-2.0.1.jar:/databricks/jars/jta-1.1.jar:/databricks/jars/jtransforms-2.4.0.jar:/databricks/jars/jul-to-slf4j-1.7.10.jar:/databricks/jars/kryo-2.21.jar:/databricks/jars/lenses_2.10-0.3.jar:/databricks/jars/leveldbjni-all-1.8.jar:/databricks/jars/libfb303-0.9.2.jar:/databricks/jars/libthrift-0.9.2.jar:/databricks/jars/lift-json_2.10-2.5.1.jar:/databricks/jars/log4j-1.2.17.jar:/databricks/jars/lz4-1.3.0.jar:/databricks/jars/mchange-commons-java-0.2.10.jar:/databricks/jars/mesos-0.21.1-shaded-protobuf.jar:/databricks/jars/metrics-core-3.0.2.jar:/databricks/jars/metrics-core-3.1.2.jar:/databricks/jars/metrics-ganglia-3.0.2.jar:/databricks/jars/metrics-ganglia-3.1.2.jar:/databricks/jars/metrics-graphite-3.1.2.jar:/databricks/jars/metrics-healthchecks-3.0.2.jar:/databricks/jars/metrics-jdbi-3.0.2.jar:/databricks/jars/metrics-jetty8-3.0.2.jar:/databricks/jars/metrics-json-3.0.2.jar:/databricks/jars/metrics-json-3.1.2.jar:/databricks/jars/metrics-jvm-3.0.2.jar:/databricks/jars/metrics-jvm-3.1.2.jar:/databricks/jars/metrics-log4j-3.0.2.jar:/databricks/jars/metrics-servlets-3.0.2.jar:/databricks/jars/minlog-1.2.jar:/databricks/jars/mysql-connector-java-5.1.27.jar:/databricks/jars/native_ref-java-1.1.jar:/databricks/jars/native_system-java-1.1.jar:/databricks/jars/netlib-native_ref-linux-armhf-1.1-natives.jar:/databricks/jars/netlib-native_ref-linux-i686-1.1-natives.jar:/databricks/jars/netlib-native_ref-linux-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_ref-osx-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_ref-win-i686-1.1-natives.jar:/databricks/jars/netlib-native_ref-win-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_system-linux-armhf-1.1-natives.jar:/databricks/jars/netlib-native_system-linux-i686-1.1-natives.jar:/databricks/jars/netlib-native_system-linux-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_system-osx-x86_64-1.1-natives.jar:/databricks/jars/netlib-native_system-win-i686-1.1-natives.jar:/databricks/jars/netlib-native_system-win-x86_64-1.1-natives.jar:/databricks/jars/netty-3.8.0.Final.jar:/databricks/jars/netty-all-4.0.29.Final.jar:/databricks/jars/objenesis-1.2.jar:/databricks/jars/oncrpc-1.0.7.jar:/databricks/jars/opencsv-2.3.jar:/databricks/jars/org.apache.jasper.glassfish-2.1.0.v201110031002.jar:/databricks/jars/org.apache.taglibs.standard.glassfish-1.2.0.v201112081803.jar:/databricks/jars/org.eclipse.jdt.core-3.7.1.jar:/databricks/jars/oro-2.0.8.jar:/databricks/jars/paranamer-2.6.jar:/databricks/jars/parquet-column-1.7.0.jar:/databricks/jars/parquet-common-1.7.0.jar:/databricks/jars/parquet-encoding-1.7.0.jar:/databricks/jars/parquet-format-2.3.0-incubating.jar:/databricks/jars/parquet-generator-1.7.0.jar:/databricks/jars/parquet-hadoop-1.7.0.jar:/databricks/jars/parquet-hadoop-bundle-1.6.0.jar:/databricks/jars/parquet-jackson-1.7.0.jar:/databricks/jars/pmml-agent-1.1.15.jar:/databricks/jars/pmml-model-1.1.15.jar:/databricks/jars/pmml-schema-1.1.15.jar:/databricks/jars/postgresql-9.4-1204-jdbc41.jar:/databricks/jars/protobuf-java-2.5.0-spark.jar:/databricks/jars/py4j-0.9.jar:/databricks/jars/pyrolite-4.9.jar:/databricks/jars/quasiquotes_2.10-2.0.0.jar:/databricks/jars/reflectasm-1.07-shaded.jar:/databricks/jars/s3_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/scala-compiler.jar:/databricks/jars/scala-library.jar:/databricks/jars/scala-reflect.jar:/databricks/jars/scalalogging-slf4j_2.10-1.1.0.jar:/databricks/jars/scalap-2.10.5.jar:/databricks/jars/scalapb-runtime_2.10-0.4.15-9.jar:/databricks/jars/slf4j-api-1.7.10.jar:/databricks/jars/slf4j-log4j12-1.7.10.jar:/databricks/jars/snappy-0.2.jar:/databricks/jars/snappy-java-1.1.2.jar:/databricks/jars/spark-avro_2.10-2.0.1.jar:/databricks/jars/spark-bagel_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-catalyst_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-core_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-csv_2.10-1.3.0.jar:/databricks/jars/spark-ganglia-lgpl_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-graphx_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-hive-thriftserver_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-hive_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-launcher_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-mllib_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-network-common_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-network-shuffle_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-redshift_2.10-0.6.0.jar:/databricks/jars/spark-repl_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-sql_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-streaming-twitter_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-streaming_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spark-unsafe_2.10-1.6-cabba801f365660179882764f7fc43b414add7e2.jar:/databricks/jars/spire-macros_2.10-0.7.4.jar:/databricks/jars/spire_2.10-0.7.4.jar:/databricks/jars/spring-core-4.1.4.RELEASE.jar:/databricks/jars/spring-test-4.1.4.RELEASE.jar:/databricks/jars/sqlite-jdbc-3.8.11.2.jar:/databricks/jars/stax-api-1.0.1.jar:/databricks/jars/stream-2.7.0.jar:/databricks/jars/stringtemplate-3.2.1.jar:/databricks/jars/super-csv-2.2.0.jar:/databricks/jars/tachyon-0.5.0-0.jar:/databricks/jars/twitter4j-core-4.0.4.jar:/databricks/jars/twitter4j-stream-4.0.4.jar:/databricks/jars/uncommons-maths-1.2.2a.jar:/databricks/jars/univocity-parsers-1.5.1.jar:/databricks/jars/unused-1.0.0.jar:/databricks/jars/util-app_2.10-6.23.0.jar:/databricks/jars/util-core_2.10-6.23.0.jar:/databricks/jars/util-jvm_2.10-6.23.0.jar:/databricks/jars/validation-api-1.1.0.Final.jar:/databricks/jars/vigdis_2.10-1.3.2-SNAPSHOT.jar:/databricks/jars/xbean-asm5-shaded-4.4.jar:/databricks/jars/xmlenc-0.52.jar:/databricks/jars/xz-1.0.jar:/databricks/jars/zookeeper-3.4.5.jar\nSUDO_GID=0\nSPARK_SCALA_VERSION=2.10\ncontainer=lxc\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\"><console>:34: error: not found: value echo\n echo $HOME\n ^\n</div>","error":null,"startTime":1.45584190696E12,"submitTime":1.455841905862E12,"finishTime":1.455841907042E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"paul.brouwers@canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"3839d22b-8a1e-48df-84ed-a5d0bc2afef9"},{"version":"CommandV1","origId":6496,"guid":"a28f1993-025d-4f02-b999-7fca99a11ac6","subtype":"command","commandType":"auto","position":9.5,"command":"%sh\ntar zxvf sou.tar.gz","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">sou/\nsou/18111105.txt\nsou/20040120.txt\nsou/19061203.txt\nsou/18411207.txt\nsou/19091207.txt\nsou/18701205.txt\nsou/19410106.txt\nsou/18571208.txt\nsou/18891203.txt\nsou/18341201.txt\nsou/19660112.txt\nsou/17981208.txt\nsou/19610130.txt\nsou/18140920.txt\nsou/18011208.txt\nsou/18811206.txt\nsou/18281202.txt\nsou/19840125.txt\nsou/18611203.txt\nsou/18731201.txt\nsou/19400103.txt\nsou/19630114.txt\nsou/19281204.txt\nsou/19221208.txt\nsou/19031207.txt\nsou/18681209.txt\nsou/18431206.txt\nsou/18861206.txt\nsou/19261207.txt\nsou/19271206.txt\nsou/19141208.txt\nsou/18791201.txt\nsou/19131202.txt\nsou/19041206.txt\nsou/18001111.txt\nsou/18041108.txt\nsou/20010227.txt\nsou/18621201.txt\nsou/19251208.txt\nsou/19700122.txt\nsou/19790125.txt\nsou/19870127.txt\nsou/20050202.txt\nsou/18331203.txt\nsou/17961207.txt\nsou/18021215.txt\nsou/18771203.txt\nsou/19890209.txt\nsou/18301206.txt\nsou/18121104.txt\nsou/19580109.txt\nsou/20110125.txt\nsou/19450106.txt\nsou/18031017.txt\nsou/19301202.txt\nsou/18661203.txt\nsou/19520109.txt\nsou/19620111.txt\nsou/18531205.txt\nsou/19610112.txt\nsou/19430107.txt\nsou/19960123.txt\nsou/17911025.txt\nsou/18211203.txt\nsou/18951207.txt\nsou/18901201.txt\nsou/18721202.txt\nsou/20140128.txt\nsou/18361205.txt\nsou/18101205.txt\nsou/18081108.txt\nsou/18961204.txt\nsou/18871206.txt\nsou/18781202.txt\nsou/19480107.txt\nsou/19001203.txt\nsou/18421206.txt\nsou/18241207.txt\nsou/18131207.txt\nsou/19500104.txt\nsou/20010920.txt\nsou/19940125.txt\nsou/19850206.txt\nsou/18541204.txt\nsou/17921106.txt\nsou/19800121.txt\nsou/19311208.txt\nsou/18461208.txt\nsou/19161205.txt\nsou/19121203.txt\nsou/19370106.txt\nsou/19151207.txt\nsou/19051205.txt\nsou/19021202.txt\nsou/18321204.txt\nsou/18671203.txt\nsou/18651204.txt\nsou/19510108.txt\nsou/18581206.txt\nsou/18161203.txt\nsou/19390104.txt\nsou/19321206.txt\nsou/18641206.txt\nsou/20070123.txt\nsou/18691206.txt\nsou/17991203.txt\nsou/18551231.txt\nsou/19440111.txt\nsou/19910129.txt\nsou/18921206.txt\nsou/18061202.txt\nsou/19470106.txt\nsou/19590109.txt\nsou/18151205.txt\nsou/18751207.txt\nsou/18981205.txt\nsou/20090224.txt\nsou/18071027.txt\nsou/18171212.txt\nsou/18821204.txt\nsou/19211206.txt\nsou/18371205.txt\nsou/19181202.txt\nsou/19720120.txt\nsou/18601203.txt\nsou/19530107.txt\nsou/18741207.txt\nsou/19460121.txt\nsou/19350104.txt\nsou/19201207.txt\nsou/18591219.txt\nsou/18221203.txt\nsou/19231206.txt\nsou/19730202.txt\nsou/19291203.txt\nsou/19820126.txt\nsou/20060131.txt\nsou/18501202.txt\nsou/17931203.txt\nsou/18711204.txt\nsou/18441203.txt\nsou/17900108.txt\nsou/18561202.txt\nsou/18381203.txt\nsou/19071203.txt\nsou/19570110.txt\nsou/19171204.txt\nsou/18181116.txt\nsou/19420106.txt\nsou/18201114.txt\nsou/20130212.txt\nsou/20030128.txt\nsou/19340103.txt\nsou/19970204.txt\nsou/19810116.txt\nsou/18841201.txt\nsou/19101206.txt\nsou/18351207.txt\nsou/17951208.txt\nsou/19530202.txt\nsou/18971206.txt\nsou/18231202.txt\nsou/18831204.txt\nsou/19490105.txt\nsou/18991205.txt\nsou/18391202.txt\nsou/18481205.txt\nsou/18631208.txt\nsou/19930217.txt\nsou/19111205.txt\nsou/19740130.txt\nsou/20160112.txt\nsou/19880125.txt\nsou/19690114.txt\nsou/19360103.txt\nsou/20120124.txt\nsou/18091129.txt\nsou/19680117.txt\nsou/18851208.txt\nsou/20100127.txt\nsou/19191202.txt\nsou/19670110.txt\nsou/18451202.txt\nsou/19241203.txt\nsou/20080128.txt\nsou/18291208.txt\nsou/19980127.txt\nsou/18401205.txt\nsou/18471207.txt\nsou/19540107.txt\nsou/18191207.txt\nsou/18801206.txt\nsou/18491204.txt\nsou/18881203.txt\nsou/19950124.txt\nsou/19380103.txt\nsou/18761205.txt\nsou/18931203.txt\nsou/18051203.txt\nsou/18271204.txt\nsou/19900131.txt\nsou/18941202.txt\nsou/20150120.txt\nsou/17941119.txt\nsou/18261205.txt\nsou/19710122.txt\nsou/17901208.txt\nsou/19081208.txt\nsou/19550106.txt\nsou/17971122.txt\nsou/19560105.txt\nsou/19640108.txt\nsou/20020129.txt\nsou/19920128.txt\nsou/18511202.txt\nsou/18311206.txt\nsou/19770112.txt\nsou/18521206.txt\nsou/19760119.txt\nsou/18251206.txt\nsou/19860204.txt\nsou/19830125.txt\nsou/19780119.txt\nsou/19650104.txt\nsou/19990119.txt\nsou/20000127.txt\nsou/19600107.txt\nsou/19011203.txt\nsou/19750115.txt\nsou/18911209.txt\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\"><console>:34: error: not found: value tar\n tar zxvf sou.tar.gz\n ^\n</div>","error":null,"startTime":1.455843270527E12,"submitTime":1.45584326944E12,"finishTime":1.45584327072E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":true,"iPythonMetadata":null,"nuid":"d1028598-4af0-4de4-85e3-54c4d5814ebc"},{"version":"CommandV1","origId":6499,"guid":"fc35f01f-1c22-4d58-8f7a-89a3af3c29e3","subtype":"command","commandType":"auto","position":9.75,"command":"%sh cd sou && ls","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">17900108.txt\n17901208.txt\n17911025.txt\n17921106.txt\n17931203.txt\n17941119.txt\n17951208.txt\n17961207.txt\n17971122.txt\n17981208.txt\n17991203.txt\n18001111.txt\n18011208.txt\n18021215.txt\n18031017.txt\n18041108.txt\n18051203.txt\n18061202.txt\n18071027.txt\n18081108.txt\n18091129.txt\n18101205.txt\n18111105.txt\n18121104.txt\n18131207.txt\n18140920.txt\n18151205.txt\n18161203.txt\n18171212.txt\n18181116.txt\n18191207.txt\n18201114.txt\n18211203.txt\n18221203.txt\n18231202.txt\n18241207.txt\n18251206.txt\n18261205.txt\n18271204.txt\n18281202.txt\n18291208.txt\n18301206.txt\n18311206.txt\n18321204.txt\n18331203.txt\n18341201.txt\n18351207.txt\n18361205.txt\n18371205.txt\n18381203.txt\n18391202.txt\n18401205.txt\n18411207.txt\n18421206.txt\n18431206.txt\n18441203.txt\n18451202.txt\n18461208.txt\n18471207.txt\n18481205.txt\n18491204.txt\n18501202.txt\n18511202.txt\n18521206.txt\n18531205.txt\n18541204.txt\n18551231.txt\n18561202.txt\n18571208.txt\n18581206.txt\n18591219.txt\n18601203.txt\n18611203.txt\n18621201.txt\n18631208.txt\n18641206.txt\n18651204.txt\n18661203.txt\n18671203.txt\n18681209.txt\n18691206.txt\n18701205.txt\n18711204.txt\n18721202.txt\n18731201.txt\n18741207.txt\n18751207.txt\n18761205.txt\n18771203.txt\n18781202.txt\n18791201.txt\n18801206.txt\n18811206.txt\n18821204.txt\n18831204.txt\n18841201.txt\n18851208.txt\n18861206.txt\n18871206.txt\n18881203.txt\n18891203.txt\n18901201.txt\n18911209.txt\n18921206.txt\n18931203.txt\n18941202.txt\n18951207.txt\n18961204.txt\n18971206.txt\n18981205.txt\n18991205.txt\n19001203.txt\n19011203.txt\n19021202.txt\n19031207.txt\n19041206.txt\n19051205.txt\n19061203.txt\n19071203.txt\n19081208.txt\n19091207.txt\n19101206.txt\n19111205.txt\n19121203.txt\n19131202.txt\n19141208.txt\n19151207.txt\n19161205.txt\n19171204.txt\n19181202.txt\n19191202.txt\n19201207.txt\n19211206.txt\n19221208.txt\n19231206.txt\n19241203.txt\n19251208.txt\n19261207.txt\n19271206.txt\n19281204.txt\n19291203.txt\n19301202.txt\n19311208.txt\n19321206.txt\n19340103.txt\n19350104.txt\n19360103.txt\n19370106.txt\n19380103.txt\n19390104.txt\n19400103.txt\n19410106.txt\n19420106.txt\n19430107.txt\n19440111.txt\n19450106.txt\n19460121.txt\n19470106.txt\n19480107.txt\n19490105.txt\n19500104.txt\n19510108.txt\n19520109.txt\n19530107.txt\n19530202.txt\n19540107.txt\n19550106.txt\n19560105.txt\n19570110.txt\n19580109.txt\n19590109.txt\n19600107.txt\n19610112.txt\n19610130.txt\n19620111.txt\n19630114.txt\n19640108.txt\n19650104.txt\n19660112.txt\n19670110.txt\n19680117.txt\n19690114.txt\n19700122.txt\n19710122.txt\n19720120.txt\n19730202.txt\n19740130.txt\n19750115.txt\n19760119.txt\n19770112.txt\n19780119.txt\n19790125.txt\n19800121.txt\n19810116.txt\n19820126.txt\n19830125.txt\n19840125.txt\n19850206.txt\n19860204.txt\n19870127.txt\n19880125.txt\n19890209.txt\n19900131.txt\n19910129.txt\n19920128.txt\n19930217.txt\n19940125.txt\n19950124.txt\n19960123.txt\n19970204.txt\n19980127.txt\n19990119.txt\n20000127.txt\n20010227.txt\n20010920.txt\n20020129.txt\n20030128.txt\n20040120.txt\n20050202.txt\n20060131.txt\n20070123.txt\n20080128.txt\n20090224.txt\n20100127.txt\n20110125.txt\n20120124.txt\n20130212.txt\n20140128.txt\n20150120.txt\n20160112.txt\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455843610078E12,"submitTime":1.455843608984E12,"finishTime":1.45584361013E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"47768323-d6c6-4d8f-8db6-5a2c30366c29"},{"version":"CommandV1","origId":6495,"guid":"c3126ddb-f37b-4ee0-ae44-c8ee6a89f191","subtype":"command","commandType":"auto","position":10.0,"command":"%sh head sou/17900108.txt","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">George Washington \n\nJanuary 8, 1790 \nFellow-Citizens of the Senate and House of Representatives: \nI embrace with great satisfaction the opportunity which now presents itself of congratulating you on the present favorable prospects of our public affairs. The recent accession of the important state of North Carolina to the Constitution of the United States (of which official information has been received), the rising credit and respectability of our country, the general and increasing good will toward the government of the Union, and the concord, peace, and plenty with which we are blessed are circumstances auspicious in an eminent degree to our national prosperity. \nIn resuming your consultations for the general good you can not but derive encouragement from the reflection that the measures of the last session have been as satisfactory to your constituents as the novelty and difficulty of the work allowed you to hope. Still further to realize their expectations and to secure the blessings which a gracious Providence has placed within our reach will in the course of the present important session call for the cool and deliberate exertion of your patriotism, firmness, and wisdom. \nAmong the many interesting objects which will engage your attention that of providing for the common defense will merit particular regard. To be prepared for war is one of the most effectual means of preserving peace. \nA free people ought not only to be armed, but disciplined; to which end a uniform and well-digested plan is requisite; and their safety and interest require that they should promote such manufactories as tend to render them independent of others for essential, particularly military, supplies. \nThe proper establishment of the troops which may be deemed indispensable will be entitled to mature consideration. In the arrangements which may be made respecting it it will be of importance to conciliate the comfortable support of the officers and soldiers with a due regard to economy. \nThere was reason to hope that the pacific measures adopted with regard to certain hostile tribes of Indians would have relieved the inhabitants of our southern and western frontiers from their depredations, but you will perceive from the information contained in the papers which I shall direct to be laid before you (comprehending a communication from the Commonwealth of Virginia) that we ought to be prepared to afford protection to those parts of the Union, and, if necessary, to punish aggressors. \n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455843636733E12,"submitTime":1.455843635645E12,"finishTime":1.455843636822E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"49f7da34-b93f-4445-be4f-7e5b34e8ba82"},{"version":"CommandV1","origId":6500,"guid":"49a38eaa-6de2-4720-887a-6358675c17f4","subtype":"command","commandType":"auto","position":10.5,"command":"%sh tail sou/17900108.txt","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">The advancement of agriculture, commerce, and manufactures by all proper means will not, I trust, need recommendation; but I can not forbear intimating to you the expediency of giving effectual encouragement as well to the introduction of new and useful inventions from abroad as to the exertions of skill and genius in producing them at home, and of facilitating the intercourse between the distant parts of our country by a due attention to the post-office and post-roads. \nNor am I less persuaded that you will agree with me in opinion that there is nothing which can better deserve your patronage than the promotion of science and literature. Knowledge is in every country the surest basis of public happiness. In one in which the measures of government receive their impressions so immediately from the sense of the community as in ours it is proportionably essential. \nTo the security of a free constitution it contributes in various ways--by convincing those who are intrusted with the public administration that every valuable end of government is best answered by the enlightened confidence of the people, and by teaching the people themselves to know and to value their own rights; to discern and provide against invasions of them; to distinguish between oppression and the necessary exercise of lawful authority; between burthens proceeding from a disregard to their convenience and those resulting from the inevitable exigencies of society; to discriminate the spirit of liberty from that of licentiousness-- cherishing the first, avoiding the last--and uniting a speedy but temperate vigilance against encroachments, with an inviolable respect to the laws. \nWhether this desirable object will be best promoted by affording aids to seminaries of learning already established, by the institution of a national university, or by any other expedients will be well worthy of a place in the deliberations of the legislature. \nGentlemen of the House of Representatives: \nI saw with peculiar pleasure at the close of the last session the resolution entered into by you expressive of your opinion that an adequate provision for the support of the public credit is a matter of high importance to the national honor and prosperity. In this sentiment I entirely concur; and to a perfect confidence in your best endeavors to devise such a provision as will be truly with the end I add an equal reliance on the cheerful cooperation of the other branch of the legislature. \nIt would be superfluous to specify inducements to a measure in which the character and interests of the United States are so obviously so deeply concerned, and which has received so explicit a sanction from your declaration. \nGentlemen of the Senate and House of Representatives: \nI have directed the proper officers to lay before you, respectively, such papers and estimates as regard the affairs particularly recommended to your consideration, and necessary to convey to you that information of the state of the Union which it is my duty to afford. \nThe welfare of our country is the great object to which our cares and efforts ought to be directed, and I shall derive great satisfaction from a cooperation with you in the pleasing though arduous task of insuring to our fellow citizens the blessings which they have a right to expect from a free, efficient, and equal government.\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455843970587E12,"submitTime":1.455843969493E12,"finishTime":1.455843970636E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2707d243-e07e-4e07-ad5b-41b8fe0421f3"},{"version":"CommandV1","origId":6497,"guid":"c8309e95-7acf-4428-9a7e-79fc9eea5574","subtype":"command","commandType":"auto","position":11.0,"command":"%sh head sou/20150120.txt","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">Barack Obama \n\nJanuary 20, 2015 \nMr. Speaker, Mr. Vice President, Members of Congress, my fellow Americans: We are 15 years into this new century. Fifteen years that dawned with terror touching our shores, that unfolded with a new generation fighting two long and costly wars, that saw a vicious recession spread across our Nation and the world. It has been and still is a hard time for many. \nBut tonight we turn the page. Tonight, after a breakthrough year for America, our economy is growing and creating jobs at the fastest pace since 1999. Our unemployment rate is now lower than it was before the financial crisis. More of our kids are graduating than ever before. More of our people are insured than ever before. And we are as free from the grip of foreign oil as we've been in almost 30 years. \nTonight, for the first time since 9/11, our combat mission in Afghanistan is over. Six years ago, nearly 180,000 American troops served in Iraq and Afghanistan. Today, fewer than 15,000 remain. And we salute the courage and sacrifice of every man and woman in this 9/11 generation who has served to keep us safe. We are humbled and grateful for your service. \nAmerica, for all that we have endured, for all the grit and hard work required to come back, for all the tasks that lie ahead, know this: The shadow of crisis has passed, and the State of the Union is strong. \nAt this momentwith a growing economy, shrinking deficits, bustling industry, booming energy productionwe have risen from recession freer to write our own future than any other nation on Earth. It's now up to us to choose who we want to be over the next 15 years and for decades to come. \nWill we accept an economy where only a few of us do spectacularly well? Or will we commit ourselves to an economy that generates rising incomes and chances for everyone who makes the effort? \nWill we approach the world fearful and reactive, dragged into costly conflicts that strain our military and set back our standing? Or will we lead wisely, using all elements of our power to defeat new threats and protect our planet? \n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455843527699E12,"submitTime":1.455843526612E12,"finishTime":1.455843527738E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"6e1199aa-0cb9-4ec8-925b-734a31c4dfa5"},{"version":"CommandV1","origId":6498,"guid":"cbb6e689-6aa3-44ec-afab-ee6f0e098ea3","subtype":"command","commandType":"auto","position":12.0,"command":"%sh tail sou/20150120.txt","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">We may have different takes on the events of Ferguson and New York. But surely we can understand a father who fears his son can't walk home without being harassed. And surely we can understand the wife who won't rest until the police officer she married walks through the front door at the end of his shift. And surely we can agree that it's a good thing that for the first time in 40 years, the crime rate and the incarceration rate have come down together, and use that as a starting point for Democrats and Republicans, community leaders and law enforcement, to reform America's criminal justice system so that it protects and serves all of us. \nThat's a better politics. That's how we start rebuilding trust. That's how we move this country forward. That's what the American people want. And that's what they deserve. \nI have no more campaigns to run. \nMy only agenda \nI know because I won both of them. My only agenda for the next 2 years is the same as the one I've had since the day I swore an oath on the steps of this Capitol: to do what I believe is best for America. If you share the broad vision I outlined tonight, I ask you to join me in the work at hand. If you disagree with parts of it, I hope you'll at least work with me where you do agree. And I commit to every Republican here tonight that I will not only seek out your ideas, I will seek to work with you to make this country stronger. \nBecause I want this Chamber, I want this city to reflect the truth: that for all our blind spots and shortcomings, we are a people with the strength and generosity of spirit to bridge divides, to unite in common effort, to help our neighbors, whether down the street or on the other side of the world. \nI want our actions to tell every child in every neighborhood, your life matters, and we are committed to improving your life chances, as committed as we are to working on behalf of our own kids. I want future generations to know that we are a people who see our differences as a great gift, that we're a people who value the dignity and worth of every citizen: man and woman, young and old, Black and White, Latino, Asian, immigrant, Native American, gay, straight, Americans with mental illness or physical disability. Everybody matters. I want them to grow up in a country that shows the world what we still know to be true: that we are still more than a collection of red States and blue States, that we are the United States of America. \nI want them to grow up in a country where a young mom can sit down and write a letter to her President with a story that sums up these past 6 years: "It's amazing what you can bounce back from when you have to. . . . We are a strong, tight-knit family who's made it through some very, very hard times." \nMy fellow Americans, we too are a strong, tight-knit family. We too have made it through some hard times. Fifteen years into this new century, we have picked ourselves up, dusted ourselves off, and begun again the work of remaking America. We have laid a new foundation. A brighter future is ours to write. Let's begin this new chapter together, and let's start the work right now. \nThank you. God bless you. God bless this country we love. Thank you.\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455843984541E12,"submitTime":1.455843983459E12,"finishTime":1.45584398462E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"f8891cc9-22a0-4e38-916b-52a3962e75f0"},{"version":"CommandV1","origId":6501,"guid":"6da2b29e-8b4b-4c01-a451-087b45311de4","subtype":"command","commandType":"auto","position":13.0,"command":"display(dbutils.fs.ls(\"dbfs:/\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/FileStore/","FileStore/",0.0],["dbfs:/databricks-datasets/","databricks-datasets/",0.0],["dbfs:/datasets/","datasets/",0.0],["dbfs:/tmp/","tmp/",0.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":null,"error":null,"startTime":1.455844210453E12,"submitTime":1.455844209373E12,"finishTime":1.455844211209E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"fc7301b7-3d02-45a1-9690-a634307179b5"},{"version":"CommandV1","origId":6502,"guid":"cd1c41c4-99db-4b8d-925c-cde986ce11bd","subtype":"command","commandType":"auto","position":14.0,"command":"display(dbutils.fs.ls(\"dbfs:/datasets\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/datasets/wiki-clickstream/","wiki-clickstream/",0.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":null,"error":null,"startTime":1.455844231092E12,"submitTime":1.455844230002E12,"finishTime":1.455844231362E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"521900c5-fb30-4ca7-b517-3e66990014cd"},{"version":"CommandV1","origId":6503,"guid":"bbedbab0-3233-4c14-b3b4-cb0a8d6eae8b","subtype":"command","commandType":"auto","position":15.0,"command":"dbutils.fs.mkdirs(\"dbfs:/datasets/sou\") //need not be done again!","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res4: Boolean = true\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455844312521E12,"submitTime":1.45584431144E12,"finishTime":1.455844312838E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"ff2b6544-e4d9-4d27-88cc-af9c1e4fc970"},{"version":"CommandV1","origId":6504,"guid":"60a013a8-94eb-4c13-9800-f9eca961a0d3","subtype":"command","commandType":"auto","position":16.0,"command":"display(dbutils.fs.ls(\"dbfs:/datasets\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/datasets/sou/","sou/",0.0],["dbfs:/datasets/wiki-clickstream/","wiki-clickstream/",0.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":null,"error":null,"startTime":1.455844322142E12,"submitTime":1.455844321055E12,"finishTime":1.455844322406E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2b5eac5a-3015-4a75-9fef-743701cf598a"},{"version":"CommandV1","origId":6506,"guid":"e14969b7-b635-4635-a6a0-585ecbff1458","subtype":"command","commandType":"auto","position":16.5,"command":"%sh pwd && ls","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">/databricks/driver\nderby.log\neventlogs\nlogs\nsou\nsou.tar.gz\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.45584438965E12,"submitTime":1.455844388513E12,"finishTime":1.455844389732E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"324fb359-695d-4837-b7fb-c047a2fc68f6"},{"version":"CommandV1","origId":6507,"guid":"91b58d7d-ffb5-495b-a65e-4f4576971591","subtype":"command","commandType":"auto","position":16.75,"command":"dbutils.fs.help","commandVersion":0,"state":"finished","results":{"type":"htmlSandbox","data":"<div class = \"ansiout\"><b>dbutils.fs</b> provides utilities for working with FileSystems. Most methods in\nthis package can take either a DBFS path (e.g., \"/foo\"), an S3 URI (\"s3n://bucket/\"), or\nanother Hadoop FileSystem URI.\n\nFor more info about a method, use <b>dbutils.fs.help(\"methodName\")</b>.\n\nIn notebooks, you can also use the %fs shorthand to access DBFS. The %fs shorthand maps\nstraightforwardly onto dbutils calls. For example, \"%fs head --maxBytes=10000 /file/path\"\ntranslates into \"dbutils.fs.head(\"/file/path\", maxBytes = 10000)\".\n <h3>fsutils</h3><b>cp(from: String, to: String, recurse: boolean = false): boolean</b> -> Copies a file or directory, possibly across FileSystems<br /><b>head(file: String, maxBytes: int = 65536): String</b> -> Returns up to the first 'maxBytes' bytes of the given file as a String encoded in UTF-8<br /><b>ls(dir: String): SchemaSeq</b> -> Lists the contents of a directory<br /><b>mkdirs(dir: String): boolean</b> -> Creates the given directory if it does not exist, also creating any necessary parent directories<br /><b>mv(from: String, to: String, recurse: boolean = false): boolean</b> -> Moves a file or directory, possibly across FileSystems<br /><b>put(file: String, contents: String, overwrite: boolean = false): boolean</b> -> Writes the given String out to a file, encoded in UTF-8<br /><b>rm(dir: String, recurse: boolean = false): boolean</b> -> Removes a file or directory<br /><br /><h3>cache</h3><b>cacheFiles(files: Seq): boolean</b> -> Caches a set of files on the local SSDs of this cluster<br /><b>cacheTable(tableName: String): boolean</b> -> Caches the contents of the given table on the local SSDs of this cluster<br /><b>uncacheFiles(files: Seq): boolean</b> -> Removes the cached version of the files<br /><b>uncacheTable(tableName: String): boolean</b> -> Removes the cached version of the given table from SSDs<br /><br /><h3>mount</h3><b>chmod(path: String, user: String, permission: String): void</b> -> Modifies the permissions of a mount point<br /><b>grants(path: String): SchemaSeq</b> -> Lists the permissions associated with a mount point<br /><b>mount(source: String, mountPoint: String, encryptionType: String = \"\", owner: String = null): boolean</b> -> Mounts the given source directory into DBFS at the given mount point<br /><b>mounts(): SchemaSeq</b> -> Displays information about what is mounted within DBFS<br /><b>refreshMounts(): boolean</b> -> Forces all machines in this cluster to refresh their mount cache, ensuring they receive the most recent information<br /><b>unmount(mountPoint: String): boolean</b> -> Deletes a DBFS mount point<br /><br /></div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455844445962E12,"submitTime":1.455844444878E12,"finishTime":1.455844446056E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"795fdb72-5aa0-4c82-90f0-b0d5ac5aa271"},{"version":"CommandV1","origId":6505,"guid":"fa82b31e-e224-4ae9-bbea-9c7914f59934","subtype":"command","commandType":"auto","position":17.0,"command":"dbutils.fs.cp(\"file:/databricks/driver/sou\", \"dbfs:/datasets/sou/\",recurse=true)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res9: Boolean = true\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"java.lang.IllegalArgumentException: Cannot copy directory unless recurse is set to true","error":"<div class=\"ansiout\">\tat com.databricks.backend.daemon.dbutils.FSUtils$.cp(DBUtilsCore.scala:83)\n\tat com.databricks.dbutils_v1.package$fs$.cp(dbutils_v1.scala:86)</div>","startTime":1.455844562178E12,"submitTime":1.4558445611E12,"finishTime":1.455844653772E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"4281736e-273d-492b-aeb4-85e57321800d"},{"version":"CommandV1","origId":6508,"guid":"95b33dcb-eaf1-4ef3-bf58-48fef5b79591","subtype":"command","commandType":"auto","position":18.0,"command":"display(dbutils.fs.ls(\"dbfs:/datasets/sou\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/datasets/sou/17900108.txt","17900108.txt",6725.0],["dbfs:/datasets/sou/17901208.txt","17901208.txt",8427.0],["dbfs:/datasets/sou/17911025.txt","17911025.txt",14175.0],["dbfs:/datasets/sou/17921106.txt","17921106.txt",12736.0],["dbfs:/datasets/sou/17931203.txt","17931203.txt",11668.0],["dbfs:/datasets/sou/17941119.txt","17941119.txt",17615.0],["dbfs:/datasets/sou/17951208.txt","17951208.txt",12296.0],["dbfs:/datasets/sou/17961207.txt","17961207.txt",17340.0],["dbfs:/datasets/sou/17971122.txt","17971122.txt",12473.0],["dbfs:/datasets/sou/17981208.txt","17981208.txt",13394.0],["dbfs:/datasets/sou/17991203.txt","17991203.txt",9236.0],["dbfs:/datasets/sou/18001111.txt","18001111.txt",8382.0],["dbfs:/datasets/sou/18011208.txt","18011208.txt",19342.0],["dbfs:/datasets/sou/18021215.txt","18021215.txt",13003.0],["dbfs:/datasets/sou/18031017.txt","18031017.txt",14022.0],["dbfs:/datasets/sou/18041108.txt","18041108.txt",12652.0],["dbfs:/datasets/sou/18051203.txt","18051203.txt",17190.0],["dbfs:/datasets/sou/18061202.txt","18061202.txt",17135.0],["dbfs:/datasets/sou/18071027.txt","18071027.txt",14334.0],["dbfs:/datasets/sou/18081108.txt","18081108.txt",16225.0],["dbfs:/datasets/sou/18091129.txt","18091129.txt",11050.0],["dbfs:/datasets/sou/18101205.txt","18101205.txt",15028.0],["dbfs:/datasets/sou/18111105.txt","18111105.txt",13941.0],["dbfs:/datasets/sou/18121104.txt","18121104.txt",19615.0],["dbfs:/datasets/sou/18131207.txt","18131207.txt",19532.0],["dbfs:/datasets/sou/18140920.txt","18140920.txt",12632.0],["dbfs:/datasets/sou/18151205.txt","18151205.txt",19398.0],["dbfs:/datasets/sou/18161203.txt","18161203.txt",20331.0],["dbfs:/datasets/sou/18171212.txt","18171212.txt",26236.0],["dbfs:/datasets/sou/18181116.txt","18181116.txt",26445.0],["dbfs:/datasets/sou/18191207.txt","18191207.txt",27880.0],["dbfs:/datasets/sou/18201114.txt","18201114.txt",20503.0],["dbfs:/datasets/sou/18211203.txt","18211203.txt",34364.0],["dbfs:/datasets/sou/18221203.txt","18221203.txt",28154.0],["dbfs:/datasets/sou/18231202.txt","18231202.txt",38329.0],["dbfs:/datasets/sou/18241207.txt","18241207.txt",49869.0],["dbfs:/datasets/sou/18251206.txt","18251206.txt",53992.0],["dbfs:/datasets/sou/18261205.txt","18261205.txt",46482.0],["dbfs:/datasets/sou/18271204.txt","18271204.txt",42481.0],["dbfs:/datasets/sou/18281202.txt","18281202.txt",44202.0],["dbfs:/datasets/sou/18291208.txt","18291208.txt",62923.0],["dbfs:/datasets/sou/18301206.txt","18301206.txt",90641.0],["dbfs:/datasets/sou/18311206.txt","18311206.txt",42902.0],["dbfs:/datasets/sou/18321204.txt","18321204.txt",46879.0],["dbfs:/datasets/sou/18331203.txt","18331203.txt",46991.0],["dbfs:/datasets/sou/18341201.txt","18341201.txt",80364.0],["dbfs:/datasets/sou/18351207.txt","18351207.txt",64395.0],["dbfs:/datasets/sou/18361205.txt","18361205.txt",73306.0],["dbfs:/datasets/sou/18371205.txt","18371205.txt",68927.0],["dbfs:/datasets/sou/18381203.txt","18381203.txt",69880.0],["dbfs:/datasets/sou/18391202.txt","18391202.txt",80147.0],["dbfs:/datasets/sou/18401205.txt","18401205.txt",55025.0],["dbfs:/datasets/sou/18411207.txt","18411207.txt",48792.0],["dbfs:/datasets/sou/18421206.txt","18421206.txt",49788.0],["dbfs:/datasets/sou/18431206.txt","18431206.txt",47670.0],["dbfs:/datasets/sou/18441203.txt","18441203.txt",55494.0],["dbfs:/datasets/sou/18451202.txt","18451202.txt",95894.0],["dbfs:/datasets/sou/18461208.txt","18461208.txt",107852.0],["dbfs:/datasets/sou/18471207.txt","18471207.txt",96912.0],["dbfs:/datasets/sou/18481205.txt","18481205.txt",127557.0],["dbfs:/datasets/sou/18491204.txt","18491204.txt",46003.0],["dbfs:/datasets/sou/18501202.txt","18501202.txt",49823.0],["dbfs:/datasets/sou/18511202.txt","18511202.txt",79335.0],["dbfs:/datasets/sou/18521206.txt","18521206.txt",59438.0],["dbfs:/datasets/sou/18531205.txt","18531205.txt",58031.0],["dbfs:/datasets/sou/18541204.txt","18541204.txt",61917.0],["dbfs:/datasets/sou/18551231.txt","18551231.txt",70459.0],["dbfs:/datasets/sou/18561202.txt","18561202.txt",63906.0],["dbfs:/datasets/sou/18571208.txt","18571208.txt",82051.0],["dbfs:/datasets/sou/18581206.txt","18581206.txt",98523.0],["dbfs:/datasets/sou/18591219.txt","18591219.txt",74089.0],["dbfs:/datasets/sou/18601203.txt","18601203.txt",84283.0],["dbfs:/datasets/sou/18611203.txt","18611203.txt",41587.0],["dbfs:/datasets/sou/18621201.txt","18621201.txt",50008.0],["dbfs:/datasets/sou/18631208.txt","18631208.txt",37109.0],["dbfs:/datasets/sou/18641206.txt","18641206.txt",36201.0],["dbfs:/datasets/sou/18651204.txt","18651204.txt",54781.0],["dbfs:/datasets/sou/18661203.txt","18661203.txt",44152.0],["dbfs:/datasets/sou/18671203.txt","18671203.txt",71650.0],["dbfs:/datasets/sou/18681209.txt","18681209.txt",60650.0],["dbfs:/datasets/sou/18691206.txt","18691206.txt",46099.0],["dbfs:/datasets/sou/18701205.txt","18701205.txt",52113.0],["dbfs:/datasets/sou/18711204.txt","18711204.txt",38805.0],["dbfs:/datasets/sou/18721202.txt","18721202.txt",23984.0],["dbfs:/datasets/sou/18731201.txt","18731201.txt",60406.0],["dbfs:/datasets/sou/18741207.txt","18741207.txt",55136.0],["dbfs:/datasets/sou/18751207.txt","18751207.txt",73272.0],["dbfs:/datasets/sou/18761205.txt","18761205.txt",40873.0],["dbfs:/datasets/sou/18771203.txt","18771203.txt",48620.0],["dbfs:/datasets/sou/18781202.txt","18781202.txt",48552.0],["dbfs:/datasets/sou/18791201.txt","18791201.txt",71149.0],["dbfs:/datasets/sou/18801206.txt","18801206.txt",41294.0],["dbfs:/datasets/sou/18811206.txt","18811206.txt",24189.0],["dbfs:/datasets/sou/18821204.txt","18821204.txt",19065.0],["dbfs:/datasets/sou/18831204.txt","18831204.txt",23860.0],["dbfs:/datasets/sou/18841201.txt","18841201.txt",55230.0],["dbfs:/datasets/sou/18851208.txt","18851208.txt",121030.0],["dbfs:/datasets/sou/18861206.txt","18861206.txt",92873.0],["dbfs:/datasets/sou/18871206.txt","18871206.txt",31685.0],["dbfs:/datasets/sou/18881203.txt","18881203.txt",55460.0],["dbfs:/datasets/sou/18891203.txt","18891203.txt",77944.0],["dbfs:/datasets/sou/18901201.txt","18901201.txt",69588.0],["dbfs:/datasets/sou/18911209.txt","18911209.txt",96894.0],["dbfs:/datasets/sou/18921206.txt","18921206.txt",81825.0],["dbfs:/datasets/sou/18931203.txt","18931203.txt",76786.0],["dbfs:/datasets/sou/18941202.txt","18941202.txt",97793.0],["dbfs:/datasets/sou/18951207.txt","18951207.txt",89791.0],["dbfs:/datasets/sou/18961204.txt","18961204.txt",94943.0],["dbfs:/datasets/sou/18971206.txt","18971206.txt",72748.0],["dbfs:/datasets/sou/18981205.txt","18981205.txt",123819.0],["dbfs:/datasets/sou/18991205.txt","18991205.txt",93175.0],["dbfs:/datasets/sou/19001203.txt","19001203.txt",118487.0],["dbfs:/datasets/sou/19011203.txt","19011203.txt",115838.0],["dbfs:/datasets/sou/19021202.txt","19021202.txt",57671.0],["dbfs:/datasets/sou/19031207.txt","19031207.txt",90262.0],["dbfs:/datasets/sou/19041206.txt","19041206.txt",104031.0],["dbfs:/datasets/sou/19051205.txt","19051205.txt",147449.0],["dbfs:/datasets/sou/19061203.txt","19061203.txt",138165.0],["dbfs:/datasets/sou/19071203.txt","19071203.txt",161983.0],["dbfs:/datasets/sou/19081208.txt","19081208.txt",115609.0],["dbfs:/datasets/sou/19091207.txt","19091207.txt",84749.0],["dbfs:/datasets/sou/19101206.txt","19101206.txt",42598.0],["dbfs:/datasets/sou/19111205.txt","19111205.txt",143491.0],["dbfs:/datasets/sou/19121203.txt","19121203.txt",153124.0],["dbfs:/datasets/sou/19131202.txt","19131202.txt",20536.0],["dbfs:/datasets/sou/19141208.txt","19141208.txt",25441.0],["dbfs:/datasets/sou/19151207.txt","19151207.txt",44773.0],["dbfs:/datasets/sou/19161205.txt","19161205.txt",12773.0],["dbfs:/datasets/sou/19171204.txt","19171204.txt",22077.0],["dbfs:/datasets/sou/19181202.txt","19181202.txt",31400.0],["dbfs:/datasets/sou/19191202.txt","19191202.txt",28511.0],["dbfs:/datasets/sou/19201207.txt","19201207.txt",16119.0],["dbfs:/datasets/sou/19211206.txt","19211206.txt",34334.0],["dbfs:/datasets/sou/19221208.txt","19221208.txt",35419.0],["dbfs:/datasets/sou/19231206.txt","19231206.txt",41144.0],["dbfs:/datasets/sou/19241203.txt","19241203.txt",42503.0],["dbfs:/datasets/sou/19251208.txt","19251208.txt",66289.0],["dbfs:/datasets/sou/19261207.txt","19261207.txt",62608.0],["dbfs:/datasets/sou/19271206.txt","19271206.txt",54125.0],["dbfs:/datasets/sou/19281204.txt","19281204.txt",50110.0],["dbfs:/datasets/sou/19291203.txt","19291203.txt",68959.0],["dbfs:/datasets/sou/19301202.txt","19301202.txt",29041.0],["dbfs:/datasets/sou/19311208.txt","19311208.txt",36649.0],["dbfs:/datasets/sou/19321206.txt","19321206.txt",26421.0],["dbfs:/datasets/sou/19340103.txt","19340103.txt",13545.0],["dbfs:/datasets/sou/19350104.txt","19350104.txt",21221.0],["dbfs:/datasets/sou/19360103.txt","19360103.txt",22300.0],["dbfs:/datasets/sou/19370106.txt","19370106.txt",16738.0],["dbfs:/datasets/sou/19380103.txt","19380103.txt",28069.0],["dbfs:/datasets/sou/19390104.txt","19390104.txt",22563.0],["dbfs:/datasets/sou/19400103.txt","19400103.txt",18722.0],["dbfs:/datasets/sou/19410106.txt","19410106.txt",19386.0],["dbfs:/datasets/sou/19420106.txt","19420106.txt",19911.0],["dbfs:/datasets/sou/19430107.txt","19430107.txt",26314.0],["dbfs:/datasets/sou/19440111.txt","19440111.txt",22151.0],["dbfs:/datasets/sou/19450106.txt","19450106.txt",48891.0],["dbfs:/datasets/sou/19460121.txt","19460121.txt",174651.0],["dbfs:/datasets/sou/19470106.txt","19470106.txt",37406.0],["dbfs:/datasets/sou/19480107.txt","19480107.txt",30550.0],["dbfs:/datasets/sou/19490105.txt","19490105.txt",20792.0],["dbfs:/datasets/sou/19500104.txt","19500104.txt",30423.0],["dbfs:/datasets/sou/19510108.txt","19510108.txt",22924.0],["dbfs:/datasets/sou/19520109.txt","19520109.txt",30228.0],["dbfs:/datasets/sou/19530107.txt","19530107.txt",56767.0],["dbfs:/datasets/sou/19530202.txt","19530202.txt",43620.0],["dbfs:/datasets/sou/19540107.txt","19540107.txt",37843.0],["dbfs:/datasets/sou/19550106.txt","19550106.txt",46532.0],["dbfs:/datasets/sou/19560105.txt","19560105.txt",52138.0],["dbfs:/datasets/sou/19570110.txt","19570110.txt",25846.0],["dbfs:/datasets/sou/19580109.txt","19580109.txt",30344.0],["dbfs:/datasets/sou/19590109.txt","19590109.txt",30145.0],["dbfs:/datasets/sou/19600107.txt","19600107.txt",35099.0],["dbfs:/datasets/sou/19610112.txt","19610112.txt",40396.0],["dbfs:/datasets/sou/19610130.txt","19610130.txt",31641.0],["dbfs:/datasets/sou/19620111.txt","19620111.txt",39488.0],["dbfs:/datasets/sou/19630114.txt","19630114.txt",31666.0],["dbfs:/datasets/sou/19640108.txt","19640108.txt",18659.0],["dbfs:/datasets/sou/19650104.txt","19650104.txt",25389.0],["dbfs:/datasets/sou/19660112.txt","19660112.txt",30570.0],["dbfs:/datasets/sou/19670110.txt","19670110.txt",41668.0],["dbfs:/datasets/sou/19680117.txt","19680117.txt",28834.0],["dbfs:/datasets/sou/19690114.txt","19690114.txt",23634.0],["dbfs:/datasets/sou/19700122.txt","19700122.txt",25408.0],["dbfs:/datasets/sou/19710122.txt","19710122.txt",25793.0],["dbfs:/datasets/sou/19720120.txt","19720120.txt",23099.0],["dbfs:/datasets/sou/19730202.txt","19730202.txt",9844.0],["dbfs:/datasets/sou/19740130.txt","19740130.txt",29231.0],["dbfs:/datasets/sou/19750115.txt","19750115.txt",24801.0],["dbfs:/datasets/sou/19760119.txt","19760119.txt",29731.0],["dbfs:/datasets/sou/19770112.txt","19770112.txt",27923.0],["dbfs:/datasets/sou/19780119.txt","19780119.txt",26564.0],["dbfs:/datasets/sou/19790125.txt","19790125.txt",19544.0],["dbfs:/datasets/sou/19800121.txt","19800121.txt",20124.0],["dbfs:/datasets/sou/19810116.txt","19810116.txt",217980.0],["dbfs:/datasets/sou/19820126.txt","19820126.txt",31166.0],["dbfs:/datasets/sou/19830125.txt","19830125.txt",33255.0],["dbfs:/datasets/sou/19840125.txt","19840125.txt",29705.0],["dbfs:/datasets/sou/19850206.txt","19850206.txt",25364.0],["dbfs:/datasets/sou/19860204.txt","19860204.txt",20449.0],["dbfs:/datasets/sou/19870127.txt","19870127.txt",22334.0],["dbfs:/datasets/sou/19880125.txt","19880125.txt",28565.0],["dbfs:/datasets/sou/19890209.txt","19890209.txt",27855.0],["dbfs:/datasets/sou/19900131.txt","19900131.txt",21434.0],["dbfs:/datasets/sou/19910129.txt","19910129.txt",22433.0],["dbfs:/datasets/sou/19920128.txt","19920128.txt",26644.0],["dbfs:/datasets/sou/19930217.txt","19930217.txt",39255.0],["dbfs:/datasets/sou/19940125.txt","19940125.txt",42320.0],["dbfs:/datasets/sou/19950124.txt","19950124.txt",51325.0],["dbfs:/datasets/sou/19960123.txt","19960123.txt",36386.0],["dbfs:/datasets/sou/19970204.txt","19970204.txt",39038.0],["dbfs:/datasets/sou/19980127.txt","19980127.txt",42255.0],["dbfs:/datasets/sou/19990119.txt","19990119.txt",43592.0],["dbfs:/datasets/sou/20000127.txt","20000127.txt",44244.0],["dbfs:/datasets/sou/20010227.txt","20010227.txt",25330.0],["dbfs:/datasets/sou/20010920.txt","20010920.txt",17383.0],["dbfs:/datasets/sou/20020129.txt","20020129.txt",22653.0],["dbfs:/datasets/sou/20030128.txt","20030128.txt",31878.0],["dbfs:/datasets/sou/20040120.txt","20040120.txt",30611.0],["dbfs:/datasets/sou/20050202.txt","20050202.txt",29875.0],["dbfs:/datasets/sou/20060131.txt","20060131.txt",31449.0],["dbfs:/datasets/sou/20070123.txt","20070123.txt",31998.0],["dbfs:/datasets/sou/20080128.txt","20080128.txt",33830.0],["dbfs:/datasets/sou/20090224.txt","20090224.txt",33640.0],["dbfs:/datasets/sou/20100127.txt","20100127.txt",40980.0],["dbfs:/datasets/sou/20110125.txt","20110125.txt",39582.0],["dbfs:/datasets/sou/20120124.txt","20120124.txt",40338.0],["dbfs:/datasets/sou/20130212.txt","20130212.txt",37815.0],["dbfs:/datasets/sou/20140128.txt","20140128.txt",39625.0],["dbfs:/datasets/sou/20150120.txt","20150120.txt",38528.0],["dbfs:/datasets/sou/20160112.txt","20160112.txt",31083.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":null,"error":null,"startTime":1.45584469709E12,"submitTime":1.455844696009E12,"finishTime":1.455844697626E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"fb367de9-efd5-47ce-a298-899dd2326596"},{"version":"CommandV1","origId":6510,"guid":"04324254-f34e-4e34-9237-88ae83b35b3d","subtype":"command","commandType":"auto","position":20.0,"command":"display(dbutils.fs.ls(\"dbfs:/datasets/\"))","commandVersion":0,"state":"finished","results":{"type":"table","data":[["dbfs:/datasets/sou/","sou/",0.0],["dbfs:/datasets/wiki-clickstream/","wiki-clickstream/",0.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"path","type":"\"string\""},{"name":"name","type":"\"string\""},{"name":"size","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":null,"error":null,"startTime":1.455845772985E12,"submitTime":1.455845771904E12,"finishTime":1.455845773321E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"1d882ff0-240b-436a-aaf0-6e3b4a4eb1f6"},{"version":"CommandV1","origId":6511,"guid":"9a12f6dd-6685-4e93-9baa-ad8bb8370dee","subtype":"command","commandType":"auto","position":21.0,"command":"val sou17900108 = sc.textFile(\"dbfs:/datasets/sou/17900108.txt\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">sou17900108: org.apache.spark.rdd.RDD[String] = dbfs:/datasets/sou/17900108.txt MapPartitionsRDD[1164] at textFile at <console>:34\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455844969071E12,"submitTime":1.45584496799E12,"finishTime":1.455844969193E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"1c036981-7856-40a0-b262-a3b98822729f"},{"version":"CommandV1","origId":6514,"guid":"25433bc5-fd3c-485e-94f5-23fedad11947","subtype":"command","commandType":"auto","position":21.5,"command":"sou17900108.take(5)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res15: Array[String] = Array("George Washington ", "", "January 8, 1790 ", "Fellow-Citizens of the Senate and House of Representatives: ", "I embrace with great satisfaction the opportunity which now presents itself of congratulating you on the present favorable prospects of our public affairs. The recent accession of the important state of North Carolina to the Constitution of the United States (of which official information has been received), the rising credit and respectability of our country, the general and increasing good will toward the government of the Union, and the concord, peace, and plenty with which we are blessed are circumstances auspicious in an eminent degree to our national prosperity. ")\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455845230764E12,"submitTime":1.455845229549E12,"finishTime":1.455845231058E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cdff63f2-183d-4cc4-8ecf-a056623100f7"},{"version":"CommandV1","origId":6515,"guid":"ae75282b-8b53-414f-8412-788af76edd10","subtype":"command","commandType":"auto","position":21.75,"command":"sou17900108.collect","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res16: Array[String] = Array("George Washington ", "", "January 8, 1790 ", "Fellow-Citizens of the Senate and House of Representatives: ", "I embrace with great satisfaction the opportunity which now presents itself of congratulating you on the present favorable prospects of our public affairs. The recent accession of the important state of North Carolina to the Constitution of the United States (of which official information has been received), the rising credit and respectability of our country, the general and increasing good will toward the government of the Union, and the concord, peace, and plenty with which we are blessed are circumstances auspicious in an eminent degree to our national prosperity. ", "In resuming your consultations for the general good you can not but derive encouragement from the reflection that the measures of the last session have been as satisfactory to your constituents as the novelty and difficulty of the work allowed you to hope. Still further to realize their expectations and to secure the blessings which a gracious Providence has placed within our reach will in the course of the present important session call for the cool and deliberate exertion of your patriotism, firmness, and wisdom. ", "Among the many interesting objects which will engage your attention that of providing for the common defense will merit particular regard. To be prepared for war is one of the most effectual means of preserving peace. ", "A free people ought not only to be armed, but disciplined; to which end a uniform and well-digested plan is requisite; and their safety and interest require that they should promote such manufactories as tend to render them independent of others for essential, particularly military, supplies. ", "The proper establishment of the troops which may be deemed indispensable will be entitled to mature consideration. In the arrangements which may be made respecting it it will be of importance to conciliate the comfortable support of the officers and soldiers with a due regard to economy. ", "There was reason to hope that the pacific measures adopted with regard to certain hostile tribes of Indians would have relieved the inhabitants of our southern and western frontiers from their depredations, but you will perceive from the information contained in the papers which I shall direct to be laid before you (comprehending a communication from the Commonwealth of Virginia) that we ought to be prepared to afford protection to those parts of the Union, and, if necessary, to punish aggressors. ", "The interests of the United States require that our intercourse with other nations should be facilitated by such provisions as will enable me to fulfill my duty in that respect in the manner which circumstances may render most conducive to the public good, and to this end that the compensation to be made to the persons who may be employed should, according to the nature of their appointments, be defined by law, and a competent fund designated for defraying the expenses incident to the conduct of foreign affairs. ", "Various considerations also render it expedient that the terms on which foreigners may be admitted to the rights of citizens should be speedily ascertained by a uniform rule of naturalization. ", "Uniformity in the currency, weights, and measures of the United States is an object of great importance, and will, I am persuaded, be duly attended to. ", "The advancement of agriculture, commerce, and manufactures by all proper means will not, I trust, need recommendation; but I can not forbear intimating to you the expediency of giving effectual encouragement as well to the introduction of new and useful inventions from abroad as to the exertions of skill and genius in producing them at home, and of facilitating the intercourse between the distant parts of our country by a due attention to the post-office and post-roads. ", "Nor am I less persuaded that you will agree with me in opinion that there is nothing which can better deserve your patronage than the promotion of science and literature. Knowledge is in every country the surest basis of public happiness. In one in which the measures of government receive their impressions so immediately from the sense of the community as in ours it is proportionably essential. ", "To the security of a free constitution it contributes in various ways--by convincing those who are intrusted with the public administration that every valuable end of government is best answered by the enlightened confidence of the people, and by teaching the people themselves to know and to value their own rights; to discern and provide against invasions of them; to distinguish between oppression and the necessary exercise of lawful authority; between burthens proceeding from a disregard to their convenience and those resulting from the inevitable exigencies of society; to discriminate the spirit of liberty from that of licentiousness-- cherishing the first, avoiding the last--and uniting a speedy but temperate vigilance against encroachments, with an inviolable respect to the laws. ", "Whether this desirable object will be best promoted by affording aids to seminaries of learning already established, by the institution of a national university, or by any other expedients will be well worthy of a place in the deliberations of the legislature. ", "Gentlemen of the House of Representatives: ", "I saw with peculiar pleasure at the close of the last session the resolution entered into by you expressive of your opinion that an adequate provision for the support of the public credit is a matter of high importance to the national honor and prosperity. In this sentiment I entirely concur; and to a perfect confidence in your best endeavors to devise such a provision as will be truly with the end I add an equal reliance on the cheerful cooperation of the other branch of the legislature. ", "It would be superfluous to specify inducements to a measure in which the character and interests of the United States are so obviously so deeply concerned, and which has received so explicit a sanction from your declaration. ", "Gentlemen of the Senate and House of Representatives: ", "I have directed the proper officers to lay before you, respectively, such papers and estimates as regard the affairs particularly recommended to your consideration, and necessary to convey to you that information of the state of the Union which it is my duty to afford. ", The welfare of our country is the great object to which our cares and efforts ought to be directed, and I shall derive great satisfaction from a cooperation with you in the pleasing though arduous task of insuring to our fellow citizens the blessings which they have a right to expect from a free, efficient, and equal government.)\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455845347254E12,"submitTime":1.455845346173E12,"finishTime":1.455845347583E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"d4972585-fb3a-429b-a2af-b0f2b8c83b7d"},{"version":"CommandV1","origId":6512,"guid":"adbc2877-ff28-472b-a29b-5c547904aaca","subtype":"command","commandType":"auto","position":22.0,"command":"sou17900108.takeOrdered(5)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res14: Array[String] = Array("", "A free people ought not only to be armed, but disciplined; to which end a uniform and well-digested plan is requisite; and their safety and interest require that they should promote such manufactories as tend to render them independent of others for essential, particularly military, supplies. ", "Among the many interesting objects which will engage your attention that of providing for the common defense will merit particular regard. To be prepared for war is one of the most effectual means of preserving peace. ", "Fellow-Citizens of the Senate and House of Representatives: ", "Gentlemen of the House of Representatives: ")\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455845062476E12,"submitTime":1.455845061388E12,"finishTime":1.45584506281E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"8e4e020b-31ec-4816-abba-0b348a77e4c3"},{"version":"CommandV1","origId":6513,"guid":"a84da3dc-68c9-43ed-b43c-600d59436867","subtype":"command","commandType":"auto","position":23.0,"command":"val souAll = sc.wholeTextFiles(\"dbfs:/datasets/sou/*.txt\")","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">souAll: org.apache.spark.rdd.RDD[(String, String)] = dbfs:/datasets/sou/*.txt MapPartitionsRDD[1268] at wholeTextFiles at <console>:34\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455845466164E12,"submitTime":1.455845465085E12,"finishTime":1.455845466316E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"fa4e8db7-1fcc-4bd8-96c7-24f075650d93"},{"version":"CommandV1","origId":6516,"guid":"77d3c612-5c99-49e3-b46f-407b00101dd9","subtype":"command","commandType":"auto","position":23.5,"command":"souAll.count","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res17: Long = 230\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455845481422E12,"submitTime":1.455845480342E12,"finishTime":1.455845510637E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b6fffe57-f455-42fc-aab8-45aa7a3cbbf5"},{"version":"CommandV1","origId":6517,"guid":"1ddc27d3-3c2a-40a8-9bb8-05fb8e830b3d","subtype":"command","commandType":"auto","position":24.0,"command":"souAll.take(2)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">res19: Array[(String, String)] = \nArray((dbfs:/datasets/sou/17900108.txt,"George Washington \n\nJanuary 8, 1790 \nFellow-Citizens of the Senate and House of Representatives: \nI embrace with great satisfaction the opportunity which now presents itself of congratulating you on the present favorable prospects of our public affairs. The recent accession of the important state of North Carolina to the Constitution of the United States (of which official information has been received), the rising credit and respectability of our country, the general and increasing good will toward the government of the Union, and the concord, peace, and plenty with which we are blessed are circumstances auspicious in an eminent degree to our national prosperity. \nIn resuming your consultations for the general good you can not but derive encouragement from the reflection that the measures of the last session have been as satisfactory to your constituents as the novelty and difficulty of the work allowed you to hope. Still further to realize their expectations and to secure the blessings which a gracious Providence has placed within our reach will in the course of the present important session call for the cool and deliberate exertion of your patriotism, firmness, and wisdom. \nAmong the many interesting objects which will engage your attention that of providing for the common defense will merit particular regard. To be prepared for war is one of the most effectual means of preserving peace. \nA free people ought not only to be armed, but disciplined; to which end a uniform and well-digested plan is requisite; and their safety and interest require that they should promote such manufactories as tend to render them independent of others for essential, particularly military, supplies. \nThe proper establishment of the troops which may be deemed indispensable will be entitled to mature consideration. In the arrangements which may be made respecting it it will be of importance to conciliate the comfortable support of the officers and soldiers with a due regard to economy. \nThere was reason to hope that the pacific measures adopted with regard to certain hostile tribes of Indians would have relieved the inhabitants of our southern and western frontiers from their depredations, but you will perceive from the information contained in the papers which I shall direct to be laid before you (comprehending a communication from the Commonwealth of Virginia) that we ought to be prepared to afford protection to those parts of the Union, and, if necessary, to punish aggressors. \nThe interests of the United States require that our intercourse with other nations should be facilitated by such provisions as will enable me to fulfill my duty in that respect in the manner which circumstances may render most conducive to the public good, and to this end that the compensation to be made to the persons who may be employed should, according to the nature of their appointments, be defined by law, and a competent fund designated for defraying the expenses incident to the conduct of foreign affairs. \nVarious considerations also render it expedient that the terms on which foreigners may be admitted to the rights of citizens should be speedily ascertained by a uniform rule of naturalization. \nUniformity in the currency, weights, and measures of the United States is an object of great importance, and will, I am persuaded, be duly attended to. \nThe advancement of agriculture, commerce, and manufactures by all proper means will not, I trust, need recommendation; but I can not forbear intimating to you the expediency of giving effectual encouragement as well to the introduction of new and useful inventions from abroad as to the exertions of skill and genius in producing them at home, and of facilitating the intercourse between the distant parts of our country by a due attention to the post-office and post-roads. \nNor am I less persuaded that you will agree with me in opinion that there is nothing which can better deserve your patronage than the promotion of science and literature. Knowledge is in every country the surest basis of public happiness. In one in which the measures of government receive their impressions so immediately from the sense of the community as in ours it is proportionably essential. \nTo the security of a free constitution it contributes in various ways--by convincing those who are intrusted with the public administration that every valuable end of government is best answered by the enlightened confidence of the people, and by teaching the people themselves to know and to value their own rights; to discern and provide against invasions of them; to distinguish between oppression and the necessary exercise of lawful authority; between burthens proceeding from a disregard to their convenience and those resulting from the inevitable exigencies of society; to discriminate the spirit of liberty from that of licentiousness-- cherishing the first, avoiding the last--and uniting a speedy but temperate vigilance against encroachments, with an inviolable respect to the laws. \nWhether this desirable object will be best promoted by affording aids to seminaries of learning already established, by the institution of a national university, or by any other expedients will be well worthy of a place in the deliberations of the legislature. \nGentlemen of the House of Representatives: \nI saw with peculiar pleasure at the close of the last session the resolution entered into by you expressive of your opinion that an adequate provision for the support of the public credit is a matter of high importance to the national honor and prosperity. In this sentiment I entirely concur; and to a perfect confidence in your best endeavors to devise such a provision as will be truly with the end I add an equal reliance on the cheerful cooperation of the other branch of the legislature. \nIt would be superfluous to specify inducements to a measure in which the character and interests of the United States are so obviously so deeply concerned, and which has received so explicit a sanction from your declaration. \nGentlemen of the Senate and House of Representatives: \nI have directed the proper officers to lay before you, respectively, such papers and estimates as regard the affairs particularly recommended to your consideration, and necessary to convey to you that information of the state of the Union which it is my duty to afford. \nThe welfare of our country is the great object to which our cares and efforts ought to be directed, and I shall derive great satisfaction from a cooperation with you in the pleasing though arduous task of insuring to our fellow citizens the blessings which they have a right to expect from a free, efficient, and equal government.\n"), (dbfs:/datasets/sou/17901208.txt,"George Washington \n\nDecember 8, 1790 \nFellow-Citizens of the Senate and House of Representatives: \nIn meeting you again I feel much satisfaction in being able to repeat my congratulations on the favorable prospects which continue to distinguish our public affairs. The abundant fruits of another year have blessed our country with plenty and with the means of a flourishing commerce. \nThe progress of public credit is witnessed by a considerable rise of American stock abroad as well as at home, and the revenues allotted for this and other national purposes have been productive beyond the calculations by which they were regulated. This latter circumstance is the more pleasing, as it is not only a proof of the fertility of our resources, but as it assures us of a further increase of the national respectability and credit, and, let me add, as it bears an honorable testimony to the patriotism and integrity of the mercantile and marine part of our citizens. The punctuality of the former in discharging their engagements has been exemplary. \nIn conformity to the powers vested in me by acts of the last session, a loan of 3,000,000 florins, toward which some provisional measures had previously taken place, has been completed in Holland. As well the celerity with which it has been filled as the nature of the terms (considering the more than ordinary demand for borrowing created by the situation of Europe) give a reasonable hope that the further execution of those powers may proceed with advantage and success. The Secretary of the Treasury has my directions to communicate such further particulars as may be requisite for more precise information. \nSince your last sessions I have received communications by which it appears that the district of Kentucky, at present a part of Virginia, has concurred in certain propositions contained in a law of that State, in consequence of which the district is to become a distinct member of the Union, in case the requisite sanction of Congress be added. For this sanction application is now made. I shall cause the papers on this very transaction to be laid before you. \nThe liberality and harmony with which it has been conducted will be found to do great honor to both the parties, and the sentiments of warm attachment to the Union and its present Government expressed by our fellow citizens of Kentucky can not fail to add an affectionate concern for their particular welfare to the great national impressions under which you will decide on the case submitted to you. \nIt has been heretofore known to Congress that frequent incursions have been made on our frontier settlements by certain banditti of Indians from the northwest side of the Ohio. These, with some of the tribes dwelling on and near the Wabash, have of late been particularly active in their depredations, and being emboldened by the impunity of their crimes and aided by such parts of the neighboring tribes as could be seduced to join in their hostilities or afford them a retreat for their prisoners and plunder, they have, instead of listening to the humane invitations and overtures made on the part of the United States, renewed their violences with fresh alacrity and greater effect. The lives of a number of valuable citizens have thus been sacrificed, and some of them under circumstances peculiarly shocking, whilst others have been carried into a deplorable captivity. \nThese aggravated provocations rendered it essential to the safety of the Western settlements that the aggressors should be made sensible that the Government of the Union is not less capable of punishing their crimes than it is disposed to respect their rights and reward their attachments. As this object could not be effected by defensive measures, it became necessary to put in force the act which empowers the President to call out the militia for the protection of the frontiers, and I have accordingly authorized an expedition in which the regular troops in that quarter are combined with such drafts of militia as were deemed sufficient. The event of the measure is yet unknown to me. The Secretary of War is directed to lay before you a statement of the information on which it is founded, as well as an estimate of the expense with which it will be attended. \nThe disturbed situation of Europe, and particularly the critical posture of the great maritime powers, whilst it ought to make us the more thankful for the general peace and security enjoyed by the United States, reminds us at the same time of the circumspection with which it becomes us to preserve these blessings. It requires also that we should not overlook the tendency of a war, and even of preparations for a war, among the nations most concerned in active commerce with this country to abridge the means, and thereby at least enhance the price, of transporting its valuable productions to their markets. I recommend it to your serious reflections how far and in what mode it may be expedient to guard against embarrassments from these contingencies by such encouragements to our own navigation as will render our commerce and agriculture less dependent on foreign bottoms, which may fail us in the very moments most interesting to both of these great objects. Our fisheries and the transportation of our own produce offer us abundant means for guarding ourselves against this evil. \nYour attention seems to be not less due to that particular branch of our trade which belongs to the Mediterranean. So many circumstances unite in rendering the present state of it distressful to us that you will not think any deliberations misemployed which may lead to its relief and protection. \nThe laws you have already passed for the establishment of a judiciary system have opened the doors of justice to all descriptions of persons. You will consider in your wisdom whether improvements in that system may yet be made, and particularly whether an uniform process of execution on sentences issuing from the Federal courts be not desirable through all the States. \nThe patronage of our commerce, of our merchants and sea men, has called for the appointment of consuls in foreign countries. It seems expedient to regulate by law the exercise of that jurisdiction and those functions which are permitted them, either by express convention or by a friendly indulgence, in the places of their residence. The consular convention, too, with His Most Christian Majesty has stipulated in certain cases the aid of the national authority to his consuls established here. Some legislative provision is requisite to carry these stipulations into full effect. \nThe establishment of the militia, of a mint, of standards of weights and measures, of the post office and post roads are subjects which I presume you will resume of course, and which are abundantly urged by their own importance. \nGentlemen of the House of Representatives: \nThe sufficiency of the revenues you have established for the objects to which they are appropriated leaves no doubt that the residuary provisions will be commensurate to the other objects for which the public faith stands now pledged. Allow me, moreover, to hope that it will be a favorite policy with you, not merely to secure a payment of the interest of the debt funded, but as far and as fast as the growing resources of the country will permit to exonerate it of the principal itself. The appropriation you have made of the Western land explains your dispositions on this subject, and I am persuaded that the sooner that valuable fund can be made to contribute, along with the other means, to the actual reduction of the public debt the more salutary will the measure be to every public interest, as well as the more satisfactory to our constituents. \nGentlemen of the Senate and House of Representatives: \nIn pursuing the various and weighty business of the present session I indulge the fullest persuasion that your consultation will be equally marked with wisdom and animated by the love of your country. In whatever belongs to my duty you shall have all the cooperation which an undiminished zeal for its welfare can inspire. It will be happy for us both, and our best reward, if, by a successful administration of our respective trusts, we can make the established Government more and more instrumental in promoting the good of our fellow citizens, and more and more the object of their attachment and confidence. \nGO. WASHINGTON\n"))\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":null,"error":null,"startTime":1.455845559996E12,"submitTime":1.455845558912E12,"finishTime":1.455845560308E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"r.sainudiin@math.canterbury.ac.nz","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"fec7cd71-516e-49d6-8f2e-98364a807853"},{"version":"CommandV1","origId":6520,"guid":"d293374d-dd2a-43f8-9736-d080811d448c","subtype":"command","commandType":"auto","position":27.0,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n### prepared by [Raazesh Sainudiin](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [Sivanand Sivaram](https://www.linkedin.com/in/sivanand)\n\n*supported by* [](https://databricks.com/)\nand \n[](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"b9abd057-efd5-49a3-bf9d-5d367a57b825"}],"dashboards":[],"guid":"900cb840-5ef1-4df7-8625-be0bfae71a9a","globalVars":{},"iPythonMetadata":null,"inputWidgets":{}};</script> <script src="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js" onerror="window.mainJsLoadError = true;"></script> </head> <body> <script> if (window.mainJsLoadError) { var u = 'https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js'; var b = document.getElementsByTagName('body')[0]; var c = document.createElement('div'); c.innerHTML = ('<h1>Network Error</h1>' + '<p><b>Please check your network connection and try again.</b></p>' + '<p>Could not load a required resource: ' + u + '</p>'); c.style.margin = '30px'; c.style.padding = '20px 50px'; c.style.backgroundColor = '#f5f5f5'; c.style.borderRadius = '5px'; b.appendChild(c); } </script> </body> </html>