{"cells":[{"cell_type":"code","source":["from pyspark.sql import SparkSession"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"bb1cebcc-5589-4b12-942e-e6f199505fc2"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["spark = SparkSession.builder.appName('lr_example').getOrCreate()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"4e0e1e07-5723-4a95-8617-7a298ae21d80"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["from pyspark.ml.regression import LinearRegression"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5d8f90ec-d4c4-4487-befe-f1a6a4592333"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["data = spark.read.csv(\"dbfs:/FileStore/shared_uploads/dizhen@hsph.harvard.edu/Ecommerce_Customers.csv\",inferSchema=True,header=True)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"0867ca63-b898-4e94-bcbb-780c56d728e5"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["data.printSchema()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"86ef49b6-d3f1-460c-bcf1-dbc99a0461eb"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"root\n |-- Email: string (nullable = true)\n |-- Address: string (nullable = true)\n |-- Avatar: string (nullable = true)\n |-- Avg Session Length: double (nullable = true)\n |-- Time on App: double (nullable = true)\n |-- Time on Website: double (nullable = true)\n |-- Length of Membership: double (nullable = true)\n |-- Yearly Amount Spent: double (nullable = true)\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["root\n |-- Email: string (nullable = true)\n |-- Address: string (nullable = true)\n |-- Avatar: string (nullable = true)\n |-- Avg Session Length: double (nullable = true)\n |-- Time on App: double (nullable = true)\n |-- Time on Website: double (nullable = true)\n |-- Length of Membership: double (nullable = true)\n |-- Yearly Amount Spent: double (nullable = true)\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["data.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"9cc292f3-b5d4-4e5b-a8b2-db9bcd3c3ace"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\n|               Email|             Address|          Avatar|Avg Session Length|       Time on App|   Time on Website|Length of Membership|Yearly Amount Spent|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\n|mstephenson@ferna...|835 Frank TunnelW...|          Violet| 34.49726772511229| 12.65565114916675| 39.57766801952616|  4.0826206329529615|  587.9510539684005|\n|   hduke@hotmail.com|4547 Archer Commo...|       DarkGreen| 31.92627202636016|11.109460728682564|37.268958868297744|    2.66403418213262|  392.2049334443264|\n|    pallen@yahoo.com|24645 Valerie Uni...|          Bisque|33.000914755642675|11.330278057777512|37.110597442120856|   4.104543202376424| 487.54750486747207|\n|riverarebecca@gma...|1414 David Throug...|     SaddleBrown| 34.30555662975554|13.717513665142507| 36.72128267790313|   3.120178782748092|  581.8523440352177|\n|mstephens@davidso...|14023 Rodriguez P...|MediumAquaMarine| 33.33067252364639|12.795188551078114| 37.53665330059473|   4.446308318351434|  599.4060920457634|\n|alvareznancy@luca...|645 Martha Park A...|     FloralWhite|33.871037879341976|12.026925339755056| 34.47687762925054|   5.493507201364199|   637.102447915074|\n|katherine20@yahoo...|68388 Reyes Light...|   DarkSlateBlue| 32.02159550138701|11.366348309710526| 36.68377615286961|   4.685017246570912|  521.5721747578274|\n|  awatkins@yahoo.com|Unit 6538 Box 898...|            Aqua|32.739142938380326| 12.35195897300293| 37.37335885854755|  4.4342734348999375|  549.9041461052942|\n|vchurch@walter-ma...|860 Lee KeyWest D...|          Salmon| 33.98777289568564|13.386235275676436|37.534497341555735|  3.2734335777477144|  570.2004089636196|\n|    bonnie69@lin.biz|PSC 2734, Box 525...|           Brown|31.936548618448917|11.814128294972196| 37.14516822352819|   3.202806071553459|  427.1993848953282|\n|andrew06@peterson...|26104 Alexander G...|          Tomato|33.992572774953736|13.338975447662113| 37.22580613162114|   2.482607770510596|  492.6060127179966|\n|ryanwerner@freema...|Unit 2413 Box 034...|          Tomato| 33.87936082480498|11.584782999535266| 37.08792607098381|    3.71320920294043|  522.3374046069357|\n|   knelson@gmail.com|6705 Miller Orcha...|       RoyalBlue|29.532428967057943|10.961298400154098| 37.42021557502538|   4.046423164299585|  408.6403510726275|\n|wrightpeter@yahoo...|05302 Dunlap Ferr...|          Bisque| 33.19033404372265|12.959226091609382|36.144666700041924|   3.918541839158999|  573.4158673313865|\n|taylormason@gmail...|7773 Powell Sprin...|        DarkBlue|32.387975853153876|13.148725692056516| 36.61995708279922|   2.494543646659249|  470.4527333009554|\n| jstark@anderson.com|49558 Ramirez Roa...|            Peru|30.737720372628182|12.636606052000127|36.213763093698624|  3.3578468423262944|  461.7807421962299|\n| wjennings@gmail.com|6362 Wilson Mount...|      PowderBlue| 32.12538689728784|11.733861690857394|  34.8940927514398|  3.1361327164897803| 457.84769594494855|\n|rebecca45@hale-ba...|8982 Burton RowWi...|       OliveDrab|32.338899323067196|12.013194694014402| 38.38513659413844|   2.420806160901484| 407.70454754954415|\n|alejandro75@hotma...|64475 Andre Club ...|            Cyan|32.187812045932155|  14.7153875441565| 38.24411459434352|   1.516575580831944|  452.3156754800354|\n|samuel46@love-wes...|544 Alexander Hei...|   LightSeaGreen| 32.61785606282345|13.989592555825254|37.190503800397956|   4.064548550437977|   605.061038804892|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\n|               Email|             Address|          Avatar|Avg Session Length|       Time on App|   Time on Website|Length of Membership|Yearly Amount Spent|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\n|mstephenson@ferna...|835 Frank TunnelW...|          Violet| 34.49726772511229| 12.65565114916675| 39.57766801952616|  4.0826206329529615|  587.9510539684005|\n|   hduke@hotmail.com|4547 Archer Commo...|       DarkGreen| 31.92627202636016|11.109460728682564|37.268958868297744|    2.66403418213262|  392.2049334443264|\n|    pallen@yahoo.com|24645 Valerie Uni...|          Bisque|33.000914755642675|11.330278057777512|37.110597442120856|   4.104543202376424| 487.54750486747207|\n|riverarebecca@gma...|1414 David Throug...|     SaddleBrown| 34.30555662975554|13.717513665142507| 36.72128267790313|   3.120178782748092|  581.8523440352177|\n|mstephens@davidso...|14023 Rodriguez P...|MediumAquaMarine| 33.33067252364639|12.795188551078114| 37.53665330059473|   4.446308318351434|  599.4060920457634|\n|alvareznancy@luca...|645 Martha Park A...|     FloralWhite|33.871037879341976|12.026925339755056| 34.47687762925054|   5.493507201364199|   637.102447915074|\n|katherine20@yahoo...|68388 Reyes Light...|   DarkSlateBlue| 32.02159550138701|11.366348309710526| 36.68377615286961|   4.685017246570912|  521.5721747578274|\n|  awatkins@yahoo.com|Unit 6538 Box 898...|            Aqua|32.739142938380326| 12.35195897300293| 37.37335885854755|  4.4342734348999375|  549.9041461052942|\n|vchurch@walter-ma...|860 Lee KeyWest D...|          Salmon| 33.98777289568564|13.386235275676436|37.534497341555735|  3.2734335777477144|  570.2004089636196|\n|    bonnie69@lin.biz|PSC 2734, Box 525...|           Brown|31.936548618448917|11.814128294972196| 37.14516822352819|   3.202806071553459|  427.1993848953282|\n|andrew06@peterson...|26104 Alexander G...|          Tomato|33.992572774953736|13.338975447662113| 37.22580613162114|   2.482607770510596|  492.6060127179966|\n|ryanwerner@freema...|Unit 2413 Box 034...|          Tomato| 33.87936082480498|11.584782999535266| 37.08792607098381|    3.71320920294043|  522.3374046069357|\n|   knelson@gmail.com|6705 Miller Orcha...|       RoyalBlue|29.532428967057943|10.961298400154098| 37.42021557502538|   4.046423164299585|  408.6403510726275|\n|wrightpeter@yahoo...|05302 Dunlap Ferr...|          Bisque| 33.19033404372265|12.959226091609382|36.144666700041924|   3.918541839158999|  573.4158673313865|\n|taylormason@gmail...|7773 Powell Sprin...|        DarkBlue|32.387975853153876|13.148725692056516| 36.61995708279922|   2.494543646659249|  470.4527333009554|\n| jstark@anderson.com|49558 Ramirez Roa...|            Peru|30.737720372628182|12.636606052000127|36.213763093698624|  3.3578468423262944|  461.7807421962299|\n| wjennings@gmail.com|6362 Wilson Mount...|      PowderBlue| 32.12538689728784|11.733861690857394|  34.8940927514398|  3.1361327164897803| 457.84769594494855|\n|rebecca45@hale-ba...|8982 Burton RowWi...|       OliveDrab|32.338899323067196|12.013194694014402| 38.38513659413844|   2.420806160901484| 407.70454754954415|\n|alejandro75@hotma...|64475 Andre Club ...|            Cyan|32.187812045932155|  14.7153875441565| 38.24411459434352|   1.516575580831944|  452.3156754800354|\n|samuel46@love-wes...|544 Alexander Hei...|   LightSeaGreen| 32.61785606282345|13.989592555825254|37.190503800397956|   4.064548550437977|   605.061038804892|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["data.head()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"3ec72854-ee50-4a89-8282-90a0d766f872"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[7]: Row(Email='mstephenson@fernandez.com', Address='835 Frank TunnelWrightmouth, MI 82180-9605', Avatar='Violet', Avg Session Length=34.49726772511229, Time on App=12.65565114916675, Time on Website=39.57766801952616, Length of Membership=4.0826206329529615, Yearly Amount Spent=587.9510539684005)","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Out[7]: Row(Email='mstephenson@fernandez.com', Address='835 Frank TunnelWrightmouth, MI 82180-9605', Avatar='Violet', Avg Session Length=34.49726772511229, Time on App=12.65565114916675, Time on Website=39.57766801952616, Length of Membership=4.0826206329529615, Yearly Amount Spent=587.9510539684005)"]}}],"execution_count":0},{"cell_type":"code","source":["for item in data.head():\n    print(item)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"667d4212-5c49-4a1a-a452-d856d1b656d5"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"mstephenson@fernandez.com\n835 Frank TunnelWrightmouth, MI 82180-9605\nViolet\n34.49726772511229\n12.65565114916675\n39.57766801952616\n4.0826206329529615\n587.9510539684005\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["mstephenson@fernandez.com\n835 Frank TunnelWrightmouth, MI 82180-9605\nViolet\n34.49726772511229\n12.65565114916675\n39.57766801952616\n4.0826206329529615\n587.9510539684005\n"]}}],"execution_count":0},{"cell_type":"markdown","source":["Set up DataFrame for Machine Learning"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"75c0bd6b-3e5a-44c8-a5a7-d024900817cf"}}},{"cell_type":"code","source":["from pyspark.ml.linalg import Vectors\nfrom pyspark.ml.feature import VectorAssembler"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"24ed125a-80f2-4354-88fd-b91781dd93cf"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["data.columns"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"7064338b-5b08-43de-8eaa-f8fc8893fd44"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Out[10]: ['Email',\n 'Address',\n 'Avatar',\n 'Avg Session Length',\n 'Time on App',\n 'Time on Website',\n 'Length of Membership',\n 'Yearly Amount Spent']","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Out[10]: ['Email',\n 'Address',\n 'Avatar',\n 'Avg Session Length',\n 'Time on App',\n 'Time on Website',\n 'Length of Membership',\n 'Yearly Amount Spent']"]}}],"execution_count":0},{"cell_type":"code","source":["assembler = VectorAssembler(\n    inputCols=[\"Avg Session Length\", \"Time on App\", \n               \"Time on Website\",'Length of Membership'],\n    outputCol=\"features\")"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"ee02cda4-b249-4db9-81a3-8e9010045efe"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["output = assembler.transform(data)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"ec82a9ed-257a-4032-8e33-929be912f6a2"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["output.select(\"features\").show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"9dfd45a3-6fd7-4ab4-90af-5b3021b1e887"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+\n|            features|\n+--------------------+\n|[34.4972677251122...|\n|[31.9262720263601...|\n|[33.0009147556426...|\n|[34.3055566297555...|\n|[33.3306725236463...|\n|[33.8710378793419...|\n|[32.0215955013870...|\n|[32.7391429383803...|\n|[33.9877728956856...|\n|[31.9365486184489...|\n|[33.9925727749537...|\n|[33.8793608248049...|\n|[29.5324289670579...|\n|[33.1903340437226...|\n|[32.3879758531538...|\n|[30.7377203726281...|\n|[32.1253868972878...|\n|[32.3388993230671...|\n|[32.1878120459321...|\n|[32.6178560628234...|\n+--------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+--------------------+\n|            features|\n+--------------------+\n|[34.4972677251122...|\n|[31.9262720263601...|\n|[33.0009147556426...|\n|[34.3055566297555...|\n|[33.3306725236463...|\n|[33.8710378793419...|\n|[32.0215955013870...|\n|[32.7391429383803...|\n|[33.9877728956856...|\n|[31.9365486184489...|\n|[33.9925727749537...|\n|[33.8793608248049...|\n|[29.5324289670579...|\n|[33.1903340437226...|\n|[32.3879758531538...|\n|[30.7377203726281...|\n|[32.1253868972878...|\n|[32.3388993230671...|\n|[32.1878120459321...|\n|[32.6178560628234...|\n+--------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["output.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"e1b909a6-4828-4f2b-8778-d78cd1e77cc1"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\n|               Email|             Address|          Avatar|Avg Session Length|       Time on App|   Time on Website|Length of Membership|Yearly Amount Spent|            features|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\n|mstephenson@ferna...|835 Frank TunnelW...|          Violet| 34.49726772511229| 12.65565114916675| 39.57766801952616|  4.0826206329529615|  587.9510539684005|[34.4972677251122...|\n|   hduke@hotmail.com|4547 Archer Commo...|       DarkGreen| 31.92627202636016|11.109460728682564|37.268958868297744|    2.66403418213262|  392.2049334443264|[31.9262720263601...|\n|    pallen@yahoo.com|24645 Valerie Uni...|          Bisque|33.000914755642675|11.330278057777512|37.110597442120856|   4.104543202376424| 487.54750486747207|[33.0009147556426...|\n|riverarebecca@gma...|1414 David Throug...|     SaddleBrown| 34.30555662975554|13.717513665142507| 36.72128267790313|   3.120178782748092|  581.8523440352177|[34.3055566297555...|\n|mstephens@davidso...|14023 Rodriguez P...|MediumAquaMarine| 33.33067252364639|12.795188551078114| 37.53665330059473|   4.446308318351434|  599.4060920457634|[33.3306725236463...|\n|alvareznancy@luca...|645 Martha Park A...|     FloralWhite|33.871037879341976|12.026925339755056| 34.47687762925054|   5.493507201364199|   637.102447915074|[33.8710378793419...|\n|katherine20@yahoo...|68388 Reyes Light...|   DarkSlateBlue| 32.02159550138701|11.366348309710526| 36.68377615286961|   4.685017246570912|  521.5721747578274|[32.0215955013870...|\n|  awatkins@yahoo.com|Unit 6538 Box 898...|            Aqua|32.739142938380326| 12.35195897300293| 37.37335885854755|  4.4342734348999375|  549.9041461052942|[32.7391429383803...|\n|vchurch@walter-ma...|860 Lee KeyWest D...|          Salmon| 33.98777289568564|13.386235275676436|37.534497341555735|  3.2734335777477144|  570.2004089636196|[33.9877728956856...|\n|    bonnie69@lin.biz|PSC 2734, Box 525...|           Brown|31.936548618448917|11.814128294972196| 37.14516822352819|   3.202806071553459|  427.1993848953282|[31.9365486184489...|\n|andrew06@peterson...|26104 Alexander G...|          Tomato|33.992572774953736|13.338975447662113| 37.22580613162114|   2.482607770510596|  492.6060127179966|[33.9925727749537...|\n|ryanwerner@freema...|Unit 2413 Box 034...|          Tomato| 33.87936082480498|11.584782999535266| 37.08792607098381|    3.71320920294043|  522.3374046069357|[33.8793608248049...|\n|   knelson@gmail.com|6705 Miller Orcha...|       RoyalBlue|29.532428967057943|10.961298400154098| 37.42021557502538|   4.046423164299585|  408.6403510726275|[29.5324289670579...|\n|wrightpeter@yahoo...|05302 Dunlap Ferr...|          Bisque| 33.19033404372265|12.959226091609382|36.144666700041924|   3.918541839158999|  573.4158673313865|[33.1903340437226...|\n|taylormason@gmail...|7773 Powell Sprin...|        DarkBlue|32.387975853153876|13.148725692056516| 36.61995708279922|   2.494543646659249|  470.4527333009554|[32.3879758531538...|\n| jstark@anderson.com|49558 Ramirez Roa...|            Peru|30.737720372628182|12.636606052000127|36.213763093698624|  3.3578468423262944|  461.7807421962299|[30.7377203726281...|\n| wjennings@gmail.com|6362 Wilson Mount...|      PowderBlue| 32.12538689728784|11.733861690857394|  34.8940927514398|  3.1361327164897803| 457.84769594494855|[32.1253868972878...|\n|rebecca45@hale-ba...|8982 Burton RowWi...|       OliveDrab|32.338899323067196|12.013194694014402| 38.38513659413844|   2.420806160901484| 407.70454754954415|[32.3388993230671...|\n|alejandro75@hotma...|64475 Andre Club ...|            Cyan|32.187812045932155|  14.7153875441565| 38.24411459434352|   1.516575580831944|  452.3156754800354|[32.1878120459321...|\n|samuel46@love-wes...|544 Alexander Hei...|   LightSeaGreen| 32.61785606282345|13.989592555825254|37.190503800397956|   4.064548550437977|   605.061038804892|[32.6178560628234...|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\n|               Email|             Address|          Avatar|Avg Session Length|       Time on App|   Time on Website|Length of Membership|Yearly Amount Spent|            features|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\n|mstephenson@ferna...|835 Frank TunnelW...|          Violet| 34.49726772511229| 12.65565114916675| 39.57766801952616|  4.0826206329529615|  587.9510539684005|[34.4972677251122...|\n|   hduke@hotmail.com|4547 Archer Commo...|       DarkGreen| 31.92627202636016|11.109460728682564|37.268958868297744|    2.66403418213262|  392.2049334443264|[31.9262720263601...|\n|    pallen@yahoo.com|24645 Valerie Uni...|          Bisque|33.000914755642675|11.330278057777512|37.110597442120856|   4.104543202376424| 487.54750486747207|[33.0009147556426...|\n|riverarebecca@gma...|1414 David Throug...|     SaddleBrown| 34.30555662975554|13.717513665142507| 36.72128267790313|   3.120178782748092|  581.8523440352177|[34.3055566297555...|\n|mstephens@davidso...|14023 Rodriguez P...|MediumAquaMarine| 33.33067252364639|12.795188551078114| 37.53665330059473|   4.446308318351434|  599.4060920457634|[33.3306725236463...|\n|alvareznancy@luca...|645 Martha Park A...|     FloralWhite|33.871037879341976|12.026925339755056| 34.47687762925054|   5.493507201364199|   637.102447915074|[33.8710378793419...|\n|katherine20@yahoo...|68388 Reyes Light...|   DarkSlateBlue| 32.02159550138701|11.366348309710526| 36.68377615286961|   4.685017246570912|  521.5721747578274|[32.0215955013870...|\n|  awatkins@yahoo.com|Unit 6538 Box 898...|            Aqua|32.739142938380326| 12.35195897300293| 37.37335885854755|  4.4342734348999375|  549.9041461052942|[32.7391429383803...|\n|vchurch@walter-ma...|860 Lee KeyWest D...|          Salmon| 33.98777289568564|13.386235275676436|37.534497341555735|  3.2734335777477144|  570.2004089636196|[33.9877728956856...|\n|    bonnie69@lin.biz|PSC 2734, Box 525...|           Brown|31.936548618448917|11.814128294972196| 37.14516822352819|   3.202806071553459|  427.1993848953282|[31.9365486184489...|\n|andrew06@peterson...|26104 Alexander G...|          Tomato|33.992572774953736|13.338975447662113| 37.22580613162114|   2.482607770510596|  492.6060127179966|[33.9925727749537...|\n|ryanwerner@freema...|Unit 2413 Box 034...|          Tomato| 33.87936082480498|11.584782999535266| 37.08792607098381|    3.71320920294043|  522.3374046069357|[33.8793608248049...|\n|   knelson@gmail.com|6705 Miller Orcha...|       RoyalBlue|29.532428967057943|10.961298400154098| 37.42021557502538|   4.046423164299585|  408.6403510726275|[29.5324289670579...|\n|wrightpeter@yahoo...|05302 Dunlap Ferr...|          Bisque| 33.19033404372265|12.959226091609382|36.144666700041924|   3.918541839158999|  573.4158673313865|[33.1903340437226...|\n|taylormason@gmail...|7773 Powell Sprin...|        DarkBlue|32.387975853153876|13.148725692056516| 36.61995708279922|   2.494543646659249|  470.4527333009554|[32.3879758531538...|\n| jstark@anderson.com|49558 Ramirez Roa...|            Peru|30.737720372628182|12.636606052000127|36.213763093698624|  3.3578468423262944|  461.7807421962299|[30.7377203726281...|\n| wjennings@gmail.com|6362 Wilson Mount...|      PowderBlue| 32.12538689728784|11.733861690857394|  34.8940927514398|  3.1361327164897803| 457.84769594494855|[32.1253868972878...|\n|rebecca45@hale-ba...|8982 Burton RowWi...|       OliveDrab|32.338899323067196|12.013194694014402| 38.38513659413844|   2.420806160901484| 407.70454754954415|[32.3388993230671...|\n|alejandro75@hotma...|64475 Andre Club ...|            Cyan|32.187812045932155|  14.7153875441565| 38.24411459434352|   1.516575580831944|  452.3156754800354|[32.1878120459321...|\n|samuel46@love-wes...|544 Alexander Hei...|   LightSeaGreen| 32.61785606282345|13.989592555825254|37.190503800397956|   4.064548550437977|   605.061038804892|[32.6178560628234...|\n+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["final_data = output.select(\"features\",'Yearly Amount Spent')"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"e31f7545-b630-4e25-87a1-b1b3340a64a2"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["train_data,test_data = final_data.randomSplit([0.7,0.3])"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"8bdffaa2-a65c-47d1-a762-c93a447aa175"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["train_data.describe().show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"288afc79-1f40-4332-9e19-550d1cc921a3"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+-------+-------------------+\n|summary|Yearly Amount Spent|\n+-------+-------------------+\n|  count|                337|\n|   mean| 497.29804728982026|\n| stddev|  75.40555176703002|\n|    min| 256.67058229005585|\n|    max|  744.2218671047146|\n+-------+-------------------+\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+-------+-------------------+\n|summary|Yearly Amount Spent|\n+-------+-------------------+\n|  count|                337|\n|   mean| 497.29804728982026|\n| stddev|  75.40555176703002|\n|    min| 256.67058229005585|\n|    max|  744.2218671047146|\n+-------+-------------------+\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["test_data.describe().show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"d96c202b-61a3-4040-a23f-a3f117866128"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+-------+-------------------+\n|summary|Yearly Amount Spent|\n+-------+-------------------+\n|  count|                163|\n|   mean|  503.4820686664174|\n| stddev|  86.93796653209513|\n|    min|   266.086340948469|\n|    max|  765.5184619388373|\n+-------+-------------------+\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+-------+-------------------+\n|summary|Yearly Amount Spent|\n+-------+-------------------+\n|  count|                163|\n|   mean|  503.4820686664174|\n| stddev|  86.93796653209513|\n|    min|   266.086340948469|\n|    max|  765.5184619388373|\n+-------+-------------------+\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["lr = LinearRegression(labelCol='Yearly Amount Spent')"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6d84c9b3-25cb-4749-86be-cfa89410aa2f"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["lrModel = lr.fit(train_data,)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"826a0ce8-ca82-4b0d-abf4-dae41db31290"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["print(\"Coefficients: {} Intercept: {}\".format(lrModel.coefficients,lrModel.intercept))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"17dd5c2b-4e84-4fb5-9347-c2c1c8faf469"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Coefficients: [25.752248212757785,38.02153501848721,0.30617838938243924,61.4168452616767] Intercept: -1038.9830898047296\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Coefficients: [25.752248212757785,38.02153501848721,0.30617838938243924,61.4168452616767] Intercept: -1038.9830898047296\n"]}}],"execution_count":0},{"cell_type":"code","source":["test_results = lrModel.evaluate(test_data)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"617ba378-fba5-4aa9-b3db-1d39b40dfa7f"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["test_results.residuals.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"18208b8c-8cbb-4b15-aa8e-5d976fbf37aa"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"/databricks/spark/python/pyspark/sql/context.py:134: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n  warnings.warn(\n+-------------------+\n|          residuals|\n+-------------------+\n|-5.4490893671846266|\n| 0.2719787663319835|\n|   11.4190363646768|\n|-17.042847592942394|\n|  7.088376721801524|\n| 4.5686319988404875|\n|-12.643497447590505|\n| -20.83164192799984|\n| 11.618492212560113|\n|  19.32851684749454|\n|  4.949463884778538|\n|  3.674630199081662|\n| -8.287042728285428|\n|-0.9510774258145602|\n| 18.653189983253924|\n|-1.0136920408320975|\n|  -4.22305643607524|\n|-2.4446599228739956|\n| 0.7896275922671521|\n|   8.81866157085966|\n+-------------------+\nonly showing top 20 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["/databricks/spark/python/pyspark/sql/context.py:134: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n  warnings.warn(\n+-------------------+\n|          residuals|\n+-------------------+\n|-5.4490893671846266|\n| 0.2719787663319835|\n|   11.4190363646768|\n|-17.042847592942394|\n|  7.088376721801524|\n| 4.5686319988404875|\n|-12.643497447590505|\n| -20.83164192799984|\n| 11.618492212560113|\n|  19.32851684749454|\n|  4.949463884778538|\n|  3.674630199081662|\n| -8.287042728285428|\n|-0.9510774258145602|\n| 18.653189983253924|\n|-1.0136920408320975|\n|  -4.22305643607524|\n|-2.4446599228739956|\n| 0.7896275922671521|\n|   8.81866157085966|\n+-------------------+\nonly showing top 20 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["unlabeled_data = test_data.select('features')"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"13a97784-9ec0-4dee-bbea-0d248b9a5478"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["predictions = lrModel.transform(unlabeled_data)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"46a68aca-eec5-4bb0-a8e2-8052fed3cc87"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["print(\"RMSE: {}\".format(test_results.rootMeanSquaredError))\nprint(\"MSE: {}\".format(test_results.meanSquaredError))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"fd525bae-91a2-4a30-991e-64cf6da981d1"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"RMSE: 10.27251391143641\nMSE: 105.5245420606546\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["RMSE: 10.27251391143641\nMSE: 105.5245420606546\n"]}}],"execution_count":0}],"metadata":{"application/vnd.databricks.v1+notebook":{"notebookName":"ml-lr-regression","dashboards":[],"notebookMetadata":{"pythonIndentUnit":4},"language":"python","widgets":{},"notebookOrigID":3598965581908814}},"nbformat":4,"nbformat_minor":0}