{"paragraphs":[{"text":"import org.apache.spark.streaming._\nimport org.apache.spark.streaming.StreamingContext\nimport org.apache.spark.streaming.StreamingContext._\nimport org.apache.spark.streaming.dstream.DStream\nimport org.apache.spark.streaming.Duration\nimport org.apache.spark.streaming.Seconds\nimport com.amazonaws.auth.{DefaultAWSCredentialsProviderChain, BasicAWSCredentials}\nimport com.amazonaws.services.kinesis.AmazonKinesisClient\nimport com.amazonaws.regions.RegionUtils\nimport org.apache.spark.streaming.kinesis.KinesisUtils\nimport com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream\nimport org.apache.spark.storage.StorageLevel\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.types.{StructType,StructField,StringType}\nimport org.apache.spark.rdd.RDD\nimport org.apache.spark.sql.Row\nimport org.apache.spark.sql.hive.HiveContext\nimport org.apache.spark.sql.SaveMode.Append\n\n\nval endpointUrl = \"https://kinesis.us-east-1.amazonaws.com\"\nval credentials = new DefaultAWSCredentialsProviderChain().getCredentials()\n    require(credentials != null,\n      \"No AWS credentials found. Please specify credentials using one of the methods specified \" +\n        \"in http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/credentials.html\")\n    val kinesisClient = new AmazonKinesisClient(credentials)\n    kinesisClient.setEndpoint(\"https://kinesis.us-east-1.amazonaws.com\")\n    val numShards = kinesisClient.describeStream(\"spark-demo\").getStreamDescription().getShards().size\n\nval numStreams = numShards\nval batchInterval = Seconds(5)\nval kinesisCheckpointInterval = batchInterval\nval regionName = RegionUtils.getRegionByEndpoint(endpointUrl).getName()\n\n \nval ssc = new StreamingContext(sc, batchInterval)\n\n // Create the Kinesis DStreams\n    val kinesisStreams = (0 until numStreams).map { i =>\n      KinesisUtils.createStream(ssc, \"app-spark-demo\", \"spark-demo\", endpointUrl, regionName,InitialPositionInStream.LATEST, kinesisCheckpointInterval, StorageLevel.MEMORY_AND_DISK_2)\n    }\n\n\n// Union all the streams\nval unionStreams = ssc.union(kinesisStreams)\n\n//Schema of the incoming data on Kinesis Stream\nval schemaString = \"device_id,temperature,timestamp\"\n\n//Parse the data\nval tableSchema = StructType( schemaString.split(\",\").map(fieldName => StructField(fieldName, StringType, true)))\n\n//Processing each RDD and storing it in temporary table\n unionStreams.foreachRDD ((rdd: RDD[Array[Byte]], time: Time) => {\n  val rowRDD = rdd.map(w => Row.fromSeq(new String(w).split(\",\")))\n  val wordsDF = sqlContext.createDataFrame(rowRDD,tableSchema)\n  wordsDF.registerTempTable(\"realTimeTable\")\n}) \n\n","dateUpdated":"May 18, 2016 10:39:41 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala","tableHide":false,"editorHide":false,"title":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615184_-1364517717","id":"20160518-223015_1999024044","dateCreated":"May 18, 2016 10:30:15 PM","status":"FINISHED","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:134","dateFinished":"May 18, 2016 10:39:49 PM","dateStarted":"May 18, 2016 10:39:42 PM","title":"Spark Streaming","focus":true},{"title":"Start streaming context","text":"ssc.start()\n","dateUpdated":"May 18, 2016 10:39:56 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala","tableHide":false,"editorHide":false,"title":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615186_-1363748219","id":"20160518-223015_792264000","result":{"code":"SUCCESS","type":"TEXT","msg":""},"dateCreated":"May 18, 2016 10:30:15 PM","dateStarted":"May 18, 2016 10:39:56 PM","dateFinished":"May 18, 2016 10:40:04 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:535"},{"text":"%sql\nshow tables","dateUpdated":"May 18, 2016 10:43:03 PM","config":{"colWidth":6,"graph":{"mode":"table","height":98,"optionOpen":false,"keys":[{"name":"tableName","index":0,"aggr":"sum"}],"values":[{"name":"isTemporary","index":1,"aggr":"sum"}],"groups":[],"scatter":{"xAxis":{"name":"tableName","index":0,"aggr":"sum"},"yAxis":{"name":"isTemporary","index":1,"aggr":"sum"}}},"enabled":true,"editorMode":"ace/mode/sql","title":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615187_-1364132968","id":"20160518-223015_398266862","result":{"code":"SUCCESS","type":"TABLE","msg":"tableName\tisTemporary\nrealtimetable\ttrue\n","comment":"","msgTable":[[{"key":"isTemporary","value":"realtimetable"},{"key":"isTemporary","value":"true"}]],"columnNames":[{"name":"tableName","index":0,"aggr":"sum"},{"name":"isTemporary","index":1,"aggr":"sum"}],"rows":[["realtimetable","true"]]},"dateCreated":"May 18, 2016 10:30:15 PM","dateStarted":"May 18, 2016 10:40:36 PM","dateFinished":"May 18, 2016 10:40:36 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:597","title":"Some SQL's"},{"text":"%sql \ndesc realtimetable","dateUpdated":"May 18, 2016 10:42:44 PM","config":{"colWidth":6,"graph":{"mode":"table","height":125,"optionOpen":false,"keys":[{"name":"col_name","index":0,"aggr":"sum"}],"values":[{"name":"data_type","index":1,"aggr":"sum"}],"groups":[],"scatter":{"xAxis":{"name":"col_name","index":0,"aggr":"sum"},"yAxis":{"name":"data_type","index":1,"aggr":"sum"}}},"enabled":true,"editorMode":"ace/mode/sql"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615187_-1364132968","id":"20160518-223015_1711626440","result":{"code":"SUCCESS","type":"TABLE","msg":"col_name\tdata_type\tcomment\ndevice_id\tstring\t\ntemperature\tstring\t\ntimestamp\tstring\t\n","comment":"","msgTable":[[{"key":"data_type","value":"device_id"},{"key":"data_type","value":"string"},{"key":"data_type","value":""}],[{"key":"comment","value":"temperature"},{"key":"comment","value":"string"},{"key":"comment","value":""}],[{"value":"timestamp"},{"value":"string"},{"value":""}]],"columnNames":[{"name":"col_name","index":0,"aggr":"sum"},{"name":"data_type","index":1,"aggr":"sum"},{"name":"comment","index":2,"aggr":"sum"}],"rows":[["device_id","string",""],["temperature","string",""],["timestamp","string",""]]},"dateCreated":"May 18, 2016 10:30:15 PM","dateStarted":"May 18, 2016 10:41:43 PM","dateFinished":"May 18, 2016 10:41:43 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:844"},{"text":"%sql\nselect device_id,timestamp, avg(temperature) as avg_temp\nfrom realtimetable  \ngroup by device_id,timestamp \norder by timestamp","dateUpdated":"May 18, 2016 10:45:17 PM","config":{"colWidth":6,"graph":{"mode":"multiBarChart","height":202,"optionOpen":false,"keys":[{"name":"device_id","index":0,"aggr":"sum"}],"values":[{"name":"avg_temp","index":2,"aggr":"sum"}],"groups":[],"scatter":{"xAxis":{"name":"device_id","index":0,"aggr":"sum"},"yAxis":{"name":"timestamp","index":1,"aggr":"sum"}},"lineWithFocus":true},"enabled":true,"editorMode":"ace/mode/sql"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615187_-1364132968","id":"20160518-223015_2112105582","result":{"code":"SUCCESS","type":"TABLE","msg":"device_id\ttimestamp\tavg_temp\n146\t2016-05-18 15:40:40\t82.0\n909\t2016-05-18 15:40:40\t84.0\n967\t2016-05-18 15:40:40\t58.0\n314\t2016-05-18 15:40:40\t89.0\n719\t2016-05-18 15:40:40\t90.0\n557\t2016-05-18 15:40:40\t83.0\n112\t2016-05-18 15:40:40\t65.0\n922\t2016-05-18 15:40:40\t70.0\n64\t2016-05-18 15:40:40\t83.0\n277\t2016-05-18 15:40:40\t64.0\n321\t2016-05-18 15:40:40\t61.0\n390\t2016-05-18 15:40:40\t72.0\n844\t2016-05-18 15:40:40\t78.0\n529\t2016-05-18 15:40:40\t58.0\n457\t2016-05-18 15:40:40\t85.0\n4\t2016-05-18 15:40:40\t74.0\n333\t2016-05-18 15:40:40\t88.0\n917\t2016-05-18 15:40:40\t67.0\n660\t2016-05-18 15:40:40\t84.0\n290\t2016-05-18 15:40:40\t82.0\n964\t2016-05-18 15:40:40\t83.0\n751\t2016-05-18 15:40:40\t66.0\n525\t2016-05-18 15:40:40\t65.0\n363\t2016-05-18 15:40:40\t82.0\n999\t2016-05-18 15:40:40\t55.0\n184\t2016-05-18 15:40:40\t74.0\n352\t2016-05-18 15:40:40\t86.0\n705\t2016-05-18 15:40:40\t69.0\n656\t2016-05-18 15:40:40\t89.0\n436\t2016-05-18 15:40:40\t79.0\n786\t2016-05-18 15:40:40\t86.0\n335\t2016-05-18 15:40:40\t86.0\n668\t2016-05-18 15:40:40\t69.0\n292\t2016-05-18 15:40:40\t73.0\n807\t2016-05-18 15:40:40\t58.0\n477\t2016-05-18 15:40:40\t84.0\n899\t2016-05-18 15:40:40\t68.0\n769\t2016-05-18 15:40:40\t67.0\n156\t2016-05-18 15:40:40\t73.0\n212\t2016-05-18 15:40:40\t74.0\n730\t2016-05-18 15:40:40\t58.0\n","comment":"","msgTable":[[{"key":"timestamp","value":"146"},{"key":"timestamp","value":"2016-05-18 15:40:40"},{"key":"timestamp","value":"82.0"}],[{"key":"avg_temp","value":"909"},{"key":"avg_temp","value":"2016-05-18 15:40:40"},{"key":"avg_temp","value":"84.0"}],[{"value":"967"},{"value":"2016-05-18 15:40:40"},{"value":"58.0"}],[{"value":"314"},{"value":"2016-05-18 15:40:40"},{"value":"89.0"}],[{"value":"719"},{"value":"2016-05-18 15:40:40"},{"value":"90.0"}],[{"value":"557"},{"value":"2016-05-18 15:40:40"},{"value":"83.0"}],[{"value":"112"},{"value":"2016-05-18 15:40:40"},{"value":"65.0"}],[{"value":"922"},{"value":"2016-05-18 15:40:40"},{"value":"70.0"}],[{"value":"64"},{"value":"2016-05-18 15:40:40"},{"value":"83.0"}],[{"value":"277"},{"value":"2016-05-18 15:40:40"},{"value":"64.0"}],[{"value":"321"},{"value":"2016-05-18 15:40:40"},{"value":"61.0"}],[{"value":"390"},{"value":"2016-05-18 15:40:40"},{"value":"72.0"}],[{"value":"844"},{"value":"2016-05-18 15:40:40"},{"value":"78.0"}],[{"value":"529"},{"value":"2016-05-18 15:40:40"},{"value":"58.0"}],[{"value":"457"},{"value":"2016-05-18 15:40:40"},{"value":"85.0"}],[{"value":"4"},{"value":"2016-05-18 15:40:40"},{"value":"74.0"}],[{"value":"333"},{"value":"2016-05-18 15:40:40"},{"value":"88.0"}],[{"value":"917"},{"value":"2016-05-18 15:40:40"},{"value":"67.0"}],[{"value":"660"},{"value":"2016-05-18 15:40:40"},{"value":"84.0"}],[{"value":"290"},{"value":"2016-05-18 15:40:40"},{"value":"82.0"}],[{"value":"964"},{"value":"2016-05-18 15:40:40"},{"value":"83.0"}],[{"value":"751"},{"value":"2016-05-18 15:40:40"},{"value":"66.0"}],[{"value":"525"},{"value":"2016-05-18 15:40:40"},{"value":"65.0"}],[{"value":"363"},{"value":"2016-05-18 15:40:40"},{"value":"82.0"}],[{"value":"999"},{"value":"2016-05-18 15:40:40"},{"value":"55.0"}],[{"value":"184"},{"value":"2016-05-18 15:40:40"},{"value":"74.0"}],[{"value":"352"},{"value":"2016-05-18 15:40:40"},{"value":"86.0"}],[{"value":"705"},{"value":"2016-05-18 15:40:40"},{"value":"69.0"}],[{"value":"656"},{"value":"2016-05-18 15:40:40"},{"value":"89.0"}],[{"value":"436"},{"value":"2016-05-18 15:40:40"},{"value":"79.0"}],[{"value":"786"},{"value":"2016-05-18 15:40:40"},{"value":"86.0"}],[{"value":"335"},{"value":"2016-05-18 15:40:40"},{"value":"86.0"}],[{"value":"668"},{"value":"2016-05-18 15:40:40"},{"value":"69.0"}],[{"value":"292"},{"value":"2016-05-18 15:40:40"},{"value":"73.0"}],[{"value":"807"},{"value":"2016-05-18 15:40:40"},{"value":"58.0"}],[{"value":"477"},{"value":"2016-05-18 15:40:40"},{"value":"84.0"}],[{"value":"899"},{"value":"2016-05-18 15:40:40"},{"value":"68.0"}],[{"value":"769"},{"value":"2016-05-18 15:40:40"},{"value":"67.0"}],[{"value":"156"},{"value":"2016-05-18 15:40:40"},{"value":"73.0"}],[{"value":"212"},{"value":"2016-05-18 15:40:40"},{"value":"74.0"}],[{"value":"730"},{"value":"2016-05-18 15:40:40"},{"value":"58.0"}]],"columnNames":[{"name":"device_id","index":0,"aggr":"sum"},{"name":"timestamp","index":1,"aggr":"sum"},{"name":"avg_temp","index":2,"aggr":"sum"}],"rows":[["146","2016-05-18 15:40:40","82.0"],["909","2016-05-18 15:40:40","84.0"],["967","2016-05-18 15:40:40","58.0"],["314","2016-05-18 15:40:40","89.0"],["719","2016-05-18 15:40:40","90.0"],["557","2016-05-18 15:40:40","83.0"],["112","2016-05-18 15:40:40","65.0"],["922","2016-05-18 15:40:40","70.0"],["64","2016-05-18 15:40:40","83.0"],["277","2016-05-18 15:40:40","64.0"],["321","2016-05-18 15:40:40","61.0"],["390","2016-05-18 15:40:40","72.0"],["844","2016-05-18 15:40:40","78.0"],["529","2016-05-18 15:40:40","58.0"],["457","2016-05-18 15:40:40","85.0"],["4","2016-05-18 15:40:40","74.0"],["333","2016-05-18 15:40:40","88.0"],["917","2016-05-18 15:40:40","67.0"],["660","2016-05-18 15:40:40","84.0"],["290","2016-05-18 15:40:40","82.0"],["964","2016-05-18 15:40:40","83.0"],["751","2016-05-18 15:40:40","66.0"],["525","2016-05-18 15:40:40","65.0"],["363","2016-05-18 15:40:40","82.0"],["999","2016-05-18 15:40:40","55.0"],["184","2016-05-18 15:40:40","74.0"],["352","2016-05-18 15:40:40","86.0"],["705","2016-05-18 15:40:40","69.0"],["656","2016-05-18 15:40:40","89.0"],["436","2016-05-18 15:40:40","79.0"],["786","2016-05-18 15:40:40","86.0"],["335","2016-05-18 15:40:40","86.0"],["668","2016-05-18 15:40:40","69.0"],["292","2016-05-18 15:40:40","73.0"],["807","2016-05-18 15:40:40","58.0"],["477","2016-05-18 15:40:40","84.0"],["899","2016-05-18 15:40:40","68.0"],["769","2016-05-18 15:40:40","67.0"],["156","2016-05-18 15:40:40","73.0"],["212","2016-05-18 15:40:40","74.0"],["730","2016-05-18 15:40:40","58.0"]]},"dateCreated":"May 18, 2016 10:30:15 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:623","dateFinished":"May 18, 2016 10:40:53 PM","dateStarted":"May 18, 2016 10:40:46 PM"},{"text":"%sql\n\nselect device_id, avg(temperature) as avg_temp from realtimetable \nwhere device_id between 100 and 150 \ngroup by device_id","dateUpdated":"May 18, 2016 10:45:17 PM","config":{"colWidth":6,"graph":{"mode":"pieChart","height":193,"optionOpen":false,"keys":[{"name":"device_id","index":0,"aggr":"sum"}],"values":[{"name":"avg_temp","index":1,"aggr":"sum"}],"groups":[],"scatter":{"xAxis":{"name":"device_id","index":0,"aggr":"sum"},"yAxis":{"name":"avg_temp","index":1,"aggr":"sum"}}},"enabled":true,"editorMode":"ace/mode/sql"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615187_-1364132968","id":"20160518-223015_253831764","result":{"code":"SUCCESS","type":"TABLE","msg":"device_id\tavg_temp\n110\t70.0\n116\t78.0\n119\t58.0\n132\t79.0\n102\t60.0\n","comment":"","msgTable":[[{"key":"avg_temp","value":"110"},{"key":"avg_temp","value":"70.0"}],[{"value":"116"},{"value":"78.0"}],[{"value":"119"},{"value":"58.0"}],[{"value":"132"},{"value":"79.0"}],[{"value":"102"},{"value":"60.0"}]],"columnNames":[{"name":"device_id","index":0,"aggr":"sum"},{"name":"avg_temp","index":1,"aggr":"sum"}],"rows":[["110","70.0"],["116","78.0"],["119","58.0"],["132","79.0"],["102","60.0"]]},"dateCreated":"May 18, 2016 10:30:15 PM","dateStarted":"May 18, 2016 10:44:18 PM","dateFinished":"May 18, 2016 10:44:20 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:646"},{"text":"%sql\nselect * from realtimetable","dateUpdated":"May 18, 2016 10:42:20 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[{"name":"device_id","index":0,"aggr":"sum"}],"values":[{"name":"temperature","index":1,"aggr":"sum"}],"groups":[],"scatter":{"xAxis":{"name":"device_id","index":0,"aggr":"sum"},"yAxis":{"name":"temperature","index":1,"aggr":"sum"}}},"enabled":true,"editorMode":"ace/mode/sql"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615187_-1364132968","id":"20160518-223015_1247934606","result":{"code":"SUCCESS","type":"TABLE","msg":"device_id\ttemperature\ttimestamp\n480\t55\t2016-05-18 15:32:58\n374\t68\t2016-05-18 15:32:58\n210\t78\t2016-05-18 15:32:58\n440\t61\t2016-05-18 15:32:58\n89\t89\t2016-05-18 15:32:58\n422\t57\t2016-05-18 15:32:58\n584\t89\t2016-05-18 15:32:58\n944\t90\t2016-05-18 15:32:58\n477\t63\t2016-05-18 15:32:58\n359\t59\t2016-05-18 15:32:58\n","comment":"","msgTable":[[{"key":"temperature","value":"480"},{"key":"temperature","value":"55"},{"key":"temperature","value":"2016-05-18 15:32:58"}],[{"key":"timestamp","value":"374"},{"key":"timestamp","value":"68"},{"key":"timestamp","value":"2016-05-18 15:32:58"}],[{"value":"210"},{"value":"78"},{"value":"2016-05-18 15:32:58"}],[{"value":"440"},{"value":"61"},{"value":"2016-05-18 15:32:58"}],[{"value":"89"},{"value":"89"},{"value":"2016-05-18 15:32:58"}],[{"value":"422"},{"value":"57"},{"value":"2016-05-18 15:32:58"}],[{"value":"584"},{"value":"89"},{"value":"2016-05-18 15:32:58"}],[{"value":"944"},{"value":"90"},{"value":"2016-05-18 15:32:58"}],[{"value":"477"},{"value":"63"},{"value":"2016-05-18 15:32:58"}],[{"value":"359"},{"value":"59"},{"value":"2016-05-18 15:32:58"}]],"columnNames":[{"name":"device_id","index":0,"aggr":"sum"},{"name":"temperature","index":1,"aggr":"sum"},{"name":"timestamp","index":2,"aggr":"sum"}],"rows":[["480","55","2016-05-18 15:32:58"],["374","68","2016-05-18 15:32:58"],["210","78","2016-05-18 15:32:58"],["440","61","2016-05-18 15:32:58"],["89","89","2016-05-18 15:32:58"],["422","57","2016-05-18 15:32:58"],["584","89","2016-05-18 15:32:58"],["944","90","2016-05-18 15:32:58"],["477","63","2016-05-18 15:32:58"],["359","59","2016-05-18 15:32:58"]]},"dateCreated":"May 18, 2016 10:30:15 PM","dateStarted":"May 18, 2016 10:35:03 PM","dateFinished":"May 18, 2016 10:35:03 PM","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:666"},{"text":"ssc.stop(stopSparkContext=false, stopGracefully=true)\n","dateUpdated":"May 18, 2016 10:40:36 PM","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorHide":false,"tableHide":false,"editorMode":"ace/mode/scala","title":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1463610615187_-1364132968","id":"20160518-223015_1182436600","result":{"code":"SUCCESS","type":"TEXT","msg":""},"dateCreated":"May 18, 2016 10:30:15 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:467","dateFinished":"May 18, 2016 10:35:23 PM","dateStarted":"May 18, 2016 10:35:18 PM","title":"Gracefully stop Streaming Context","focus":true}],"name":"Spark Streaming","id":"2BMXT8YZX","angularObjects":{"2B44YVSN1":[],"2AJXGMUUJ":[],"2AK8P7CPX":[],"2AM1YV5CU":[],"2AKK3QQXU":[],"2ANGGHHMQ":[]},"config":{"looknfeel":"default"},"info":{}}