{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import findspark\n", "findspark.init()\n", "import pyspark\n", "sc = pyspark.SparkContext()" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from pyspark.sql.session import SparkSession\n", "spark = SparkSession(sc)\n", "ks_data = spark.read.option(\"header\", \"true\").csv(\"/Users/ryanshin/Downloads/ks-projects-201801.csv\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----------+--------------------+--------------+-------------+--------+----------+---------+-------------------+--------+----------+-------+-------+-----------+----------------+-------------+\n", "| ID| name| category|main_category|currency| deadline| goal| launched| pledged| state|backers|country|usd pledged|usd_pledged_real|usd_goal_real|\n", "+----------+--------------------+--------------+-------------+--------+----------+---------+-------------------+--------+----------+-------+-------+-----------+----------------+-------------+\n", "|1000002330|The Songs of Adel...| Poetry| Publishing| GBP|2015-10-09| 1000.00|2015-08-11 12:12:28| 0.00| failed| 0| GB| 0.00| 0.00| 1533.95|\n", "|1000003930|Greeting From Ear...|Narrative Film| Film & Video| USD|2017-11-01| 30000.00|2017-09-02 04:43:57| 2421.00| failed| 15| US| 100.00| 2421.00| 30000.00|\n", "|1000004038| Where is Hank?|Narrative Film| Film & Video| USD|2013-02-26| 45000.00|2013-01-12 00:20:50| 220.00| failed| 3| US| 220.00| 220.00| 45000.00|\n", "|1000007540|ToshiCapital Reko...| Music| Music| USD|2012-04-16| 5000.00|2012-03-17 03:24:11| 1.00| failed| 1| US| 1.00| 1.00| 5000.00|\n", "|1000011046|Community Film Pr...| Film & Video| Film & Video| USD|2015-08-29| 19500.00|2015-07-04 08:35:03| 1283.00| canceled| 14| US| 1283.00| 1283.00| 19500.00|\n", "|1000014025|Monarch Espresso Bar| Restaurants| Food| USD|2016-04-01| 50000.00|2016-02-26 13:38:27|52375.00|successful| 224| US| 52375.00| 52375.00| 50000.00|\n", "|1000023410|Support Solar Roa...| Food| Food| USD|2014-12-21| 1000.00|2014-12-01 18:30:44| 1205.00|successful| 16| US| 1205.00| 1205.00| 1000.00|\n", "|1000030581|Chaser Strips. Ou...| Drinks| Food| USD|2016-03-17| 25000.00|2016-02-01 20:05:12| 453.00| failed| 40| US| 453.00| 453.00| 25000.00|\n", "|1000034518|SPIN - Premium Re...|Product Design| Design| USD|2014-05-29|125000.00|2014-04-24 18:14:43| 8233.00| canceled| 58| US| 8233.00| 8233.00| 125000.00|\n", "| 100004195|STUDIO IN THE SKY...| Documentary| Film & Video| USD|2014-08-10| 65000.00|2014-07-11 21:55:48| 6240.57| canceled| 43| US| 6240.57| 6240.57| 65000.00|\n", "| 100004721| Of Jesus and Madmen| Nonfiction| Publishing| CAD|2013-10-09| 2500.00|2013-09-09 18:19:37| 0.00| failed| 0| CA| 0.00| 0.00| 2406.39|\n", "| 100005484| Lisa Lim New CD!| Indie Rock| Music| USD|2013-04-08| 12500.00|2013-03-09 06:42:58|12700.00|successful| 100| US| 12700.00| 12700.00| 12500.00|\n", "|1000055792| The Cottage Market| Crafts| Crafts| USD|2014-10-02| 5000.00|2014-09-02 17:11:50| 0.00| failed| 0| US| 0.00| 0.00| 5000.00|\n", "|1000056157|G-Spot Place for ...| Games| Games| USD|2016-03-25|200000.00|2016-02-09 23:01:12| 0.00| failed| 0| US| 0.00| 0.00| 200000.00|\n", "|1000057089|Tombstone: Old We...|Tabletop Games| Games| GBP|2017-05-03| 5000.00|2017-04-05 19:44:18|94175.00|successful| 761| GB| 57763.78| 121857.33| 6469.73|\n", "|1000064368| Survival Rings| Design| Design| USD|2015-02-28| 2500.00|2015-01-29 02:10:53| 664.00| failed| 11| US| 664.00| 664.00| 2500.00|\n", "|1000064918| The Beard| Comic Books| Comics| USD|2014-11-08| 1500.00|2014-10-09 22:27:52| 395.00| failed| 16| US| 395.00| 395.00| 1500.00|\n", "|1000068480|Notes From London...| Art Books| Publishing| USD|2015-05-10| 3000.00|2015-04-10 21:20:54| 789.00| failed| 20| US| 789.00| 789.00| 3000.00|\n", "|1000070642|Mike Corey's Dark...| Music| Music| USD|2012-08-17| 250.00|2012-08-02 14:11:32| 250.00|successful| 7| US| 250.00| 250.00| 250.00|\n", "|1000071625| Boco Tea| Food| Food| USD|2012-06-02| 5000.00|2012-05-03 17:24:32| 1781.00| failed| 40| US| 1781.00| 1781.00| 5000.00|\n", "+----------+--------------------+--------------+-------------+--------+----------+---------+-------------------+--------+----------+-------+-------+-----------+----------------+-------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ "ks_data.show()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "+----------+--------------------+--------+-------------+--------+----------+--------+-------------------+-------+----------+-------+-------+-----------+----------------+-------------+\n", "| ID| name|category|main_category|currency| deadline| goal| launched|pledged| state|backers|country|usd pledged|usd_pledged_real|usd_goal_real|\n", "+----------+--------------------+--------+-------------+--------+----------+--------+-------------------+-------+----------+-------+-------+-----------+----------------+-------------+\n", "|1000002330|The Songs of Adel...| Poetry| Publishing| GBP|2015-10-09| 1000.00|2015-08-11 12:12:28| 0.00| failed| 0| GB| 0.00| 0.00| 1533.95|\n", "|1001028167|Steel Cathedrals-...| Poetry| Publishing| GBP|2013-07-28| 4000.00|2013-06-28 23:17:04| 20.00| failed| 1| GB| 30.66| 30.30| 6060.97|\n", "|1001468086| Bass River Press| Poetry| Publishing| USD|2015-04-05| 2000.00|2015-03-06 19:58:58| 325.00| failed| 6| US| 325.00| 325.00| 2000.00|\n", "|1001991458|Poems For Apostat...| Poetry| Publishing| USD|2014-05-31|10000.00|2014-05-01 17:47:49| 101.00| failed| 2| US| 101.00| 101.00| 10000.00|\n", "|1002519316| Your poem, by me.| Poetry| Publishing| CAD|2015-10-31| 1000.00|2015-10-01 20:00:32| 55.00| failed| 3| CA| 40.98| 41.66| 757.52|\n", "|1003899141|Pretty the Ugly a...| Poetry| Publishing| USD|2014-01-20| 800.00|2014-01-14 03:40:04| 200.00| failed| 7| US| 200.00| 200.00| 800.00|\n", "|1004671073|Poetry Will Never...| Poetry| Publishing| USD|2013-02-23| 3000.00|2013-01-24 09:51:16| 987.00| failed| 35| US| 987.00| 987.00| 3000.00|\n", "|1006938422|Playdoughs Republ...| Poetry| Publishing| USD|2012-10-21| 7770.00|2012-08-22 00:52:10| 78.77| failed| 8| US| 78.77| 78.77| 7770.00|\n", "|1010227672|Book of Guidance:...| Poetry| Publishing| USD|2017-06-27| 1000.00|2017-05-28 20:18:19| 5.00| suspended| 2| US| 0.00| 5.00| 1000.00|\n", "|1010516592| A £13 Poem| Poetry| Publishing| GBP|2015-02-14| 13.00|2015-02-08 20:04:19| 125.49|successful| 17| GB| 191.26| 192.95| 19.99|\n", "|1011481501|poetry for the wo...| Poetry| Publishing| AUD|2014-10-01| 8000.00|2014-08-12 02:15:14| 100.00| failed| 1| AU| 92.77| 87.52| 7001.58|\n", "|1013466522|'What Was Felt Ba...| Poetry| Publishing| CAD|2017-10-08| 700.00|2017-09-26 06:43:31|1023.00|successful| 30| CA| 46.22| 818.47| 560.04|\n", "|1017960054|Stingrays Poetry ...| Poetry| Publishing| USD|2016-12-14| 700.00|2016-11-11 12:55:19| 826.00|successful| 27| US| 70.00| 826.00| 700.00|\n", "|1017998338|:60 Poetry Collec...| Poetry| Publishing| USD|2013-08-01| 1500.00|2013-07-02 02:31:54| 361.00| failed| 12| US| 361.00| 361.00| 1500.00|\n", "|1019096993|\"\"\"Photoetry\"\"- A...| Poetry| Publishing| USD|2013-06-06| 750.00|2013-05-07 03:32:25| 0.00| failed| 0| US| 0.00| 0.00| 750.00|\n", "|1019552425|An Adventure Poet...| Poetry| Publishing| USD|2016-09-24| 1500.00|2016-08-10 22:35:41|2851.00|successful| 84| US| 233.00| 2851.00| 1500.00|\n", "|1021233626|Wounded Hearts, W...| Poetry| Publishing| USD|2015-12-31|10000.00|2015-12-01 04:40:17| 330.00| failed| 4| US| 330.00| 330.00| 10000.00|\n", "|1021375350|\"\"\"Moments\"\" A bo...| Poetry| Publishing| USD|2014-11-21| 2700.00|2014-10-22 15:44:16| 50.00| failed| 2| US| 50.00| 50.00| 2700.00|\n", "|1025502083|Assist Sublimatio...| Poetry| Publishing| USD|2010-12-19| 1000.00|2010-11-19 14:54:26| 590.00| failed| 18| US| 590.00| 590.00| 1000.00|\n", "|1026038129| The America Poems| Poetry| Publishing| USD|2013-05-19| 500.00|2013-04-19 15:11:57| 650.00|successful| 29| US| 650.00| 650.00| 500.00|\n", "+----------+--------------------+--------+-------------+--------+----------+--------+-------------------+-------+----------+-------+-------+-----------+----------------+-------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ "from pyspark.sql.functions import col\n", "ks_data.filter(col(\"category\") == \"Poetry\").show()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from IPython.display import IFrame\n", "IFrame('https://www.zepl.com/viewer/notebooks/bm90ZTovL1NEUkx1cmtlci8wZmY1NDc4OTJjN2M0OGI5YWFkZmY5ZmIyZWFkNzgzOC9ub3RlLmpzb24', width='100%', height=600)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 2 }