{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"<driver>","Host":"clinton","Port":58181},"Maximum Memory":11976635842,"Timestamp":1419923338972}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-8-oracle/jre","Java Version":"1.8.0_25 (Oracle Corporation)","Scala Version":"version 2.10.4"},"Spark Properties":{"spark.akka.timeout":"600000","spark.driver.host":"clinton","spark.akka.askTimeout":"600000","spark.eventLog.enabled":"true","spark.driver.port":"60263","spark.jars":"file:/if7/user/Trademaker_Spark/target/scala-2.10/trademaker.jar","spark.app.name":"Trademaker","spark.scheduler.mode":"FIFO","spark.storage.memoryFraction":"0.9","spark.driver.memory":"14g","spark.default.parallelism":"16","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Xmx12g -Xms12g -Xss1g","spark.akka.retry.wait":"600000","spark.storage.blockManagerSlaveTimeoutMs":"600000","spark.master":"spark://master:7077","spark.executor.memory":"1980m","spark.eventLog.dir":"hdfs://centurion002:8020/SparkLog","spark.worker.timeout":"600000","spark.fileserver.uri":"http://master:57193","spark.tachyonStore.folderName":"spark-f695f7ca-336c-4926-9b46-e547932c2578","spark.app.id":"app-20141230020857-0001","spark.akka.frameSize":"200"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.vm.specification.version":"1.8","user.home":"/home/user","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-8-oracle/jre/lib/amd64","user.dir":"/if7/user/Trademaker_Spark","java.library.path":"/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib","sun.cpu.isalist":"","sun.desktop":"gnome","os.arch":"amd64","java.vm.version":"25.25-b02","java.endorsed.dirs":"/usr/lib/jvm/java-8-oracle/jre/lib/endorsed","java.runtime.version":"1.8.0_25-b17","java.vm.info":"mixed mode","java.ext.dirs":"/usr/lib/jvm/java-8-oracle/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"Java(TM) SE Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/lib/jvm/java-8-oracle/jre/lib/resources.jar:/usr/lib/jvm/java-8-oracle/jre/lib/rt.jar:/usr/lib/jvm/java-8-oracle/jre/lib/sunrsasign.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jsse.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jce.jar:/usr/lib/jvm/java-8-oracle/jre/lib/charsets.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jfr.jar:/usr/lib/jvm/java-8-oracle/jre/classes","file.encoding":"UTF-8","user.timezone":"America/New_York","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"3.16.0-28-generic","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"user","java.vm.name":"Java HotSpot(TM) 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://master:7077 --class Main --driver-memory 14g --driver-java-options -Xmx12g -Xms12g -Xss1g target/scala-2.10/trademaker.jar","java.home":"/usr/lib/jvm/java-8-oracle/jre","java.version":"1.8.0_25","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/if7/user/spark/assembly/target/scala-2.10/spark-assembly-1.2.0-hadoop2.5.1.jar":"System Classpath","/if7/user/spark/conf":"System Classpath","/opt/hadoop/etc/hadoop":"System Classpath","http://master:57193/jars/trademaker.jar":"Added By User"}}
{"Event":"SparkListenerApplicationStart","App Name":"Trademaker","App ID":"app-20141230020857-0001","Timestamp":1419923320332,"User":"user"}
{"Event":"SparkListenerJobStart","Job ID":0,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"collect at DBTrademaker.scala:633","Number of Tasks":16,"RDD Info":[{"RDD ID":1,"Name":"1","Storage Level":{"Use Disk":false,"Use Memory":false,"Use Tachyon":false,"Deserialized":false,"Replication":1},"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Tachyon Size":0,"Disk Size":0},{"RDD ID":0,"Name":"0","Storage Level":{"Use Disk":false,"Use Memory":false,"Use Tachyon":false,"Deserialized":false,"Replication":1},"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Tachyon Size":0,"Disk Size":0}],"Details":"org.apache.spark.rdd.RDD.collect(RDD.scala:780)\nDBTrademaker.my_analyze_MyMap(DBTrademaker.scala:633)\nDBTrademaker$$anonfun$3.apply(DBTrademaker.scala:566)\nDBTrademaker$$anonfun$3.apply(DBTrademaker.scala:566)\nFramework.TrademakerFramework$$anonfun$tradespace$1$$anonfun$apply$6.apply(TrademakerFramework.scala:97)\nFramework.TrademakerFramework$$anonfun$tradespace$1$$anonfun$apply$6.apply(TrademakerFramework.scala:96)\nDBTrademaker$$anonfun$run$1.apply(DBTrademaker.scala:74)\nDBTrademaker$$anonfun$run$1.apply(DBTrademaker.scala:72)\nscala.collection.immutable.List.foreach(List.scala:318)\nDBTrademaker.run(DBTrademaker.scala:72)\nMain$.main(Main.scala:12)\nMain.main(Main.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:483)\norg.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:353)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:75)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[]}],"Stage IDs":[0]}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"collect at DBTrademaker.scala:633","Number of Tasks":16,"RDD Info":[{"RDD ID":1,"Name":"1","Storage Level":{"Use Disk":false,"Use Memory":false,"Use Tachyon":false,"Deserialized":false,"Replication":1},"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Tachyon Size":0,"Disk Size":0},{"RDD ID":0,"Name":"0","Storage Level":{"Use Disk":false,"Use Memory":false,"Use Tachyon":false,"Deserialized":false,"Replication":1},"Number of Partitions":16,"Number of Cached Partitions":0,"Memory Size":0,"Tachyon Size":0,"Disk Size":0}],"Details":"org.apache.spark.rdd.RDD.collect(RDD.scala:780)\nDBTrademaker.my_analyze_MyMap(DBTrademaker.scala:633)\nDBTrademaker$$anonfun$3.apply(DBTrademaker.scala:566)\nDBTrademaker$$anonfun$3.apply(DBTrademaker.scala:566)\nFramework.TrademakerFramework$$anonfun$tradespace$1$$anonfun$apply$6.apply(TrademakerFramework.scala:97)\nFramework.TrademakerFramework$$anonfun$tradespace$1$$anonfun$apply$6.apply(TrademakerFramework.scala:96)\nDBTrademaker$$anonfun$run$1.apply(DBTrademaker.scala:74)\nDBTrademaker$$anonfun$run$1.apply(DBTrademaker.scala:72)\nscala.collection.immutable.List.foreach(List.scala:318)\nDBTrademaker.run(DBTrademaker.scala:72)\nMain$.main(Main.scala:12)\nMain.main(Main.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:483)\norg.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:353)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:75)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[]}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"23","Host":"centurion040","Port":33425},"Maximum Memory":1611635097,"Timestamp":1419923345664}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"21","Host":"centurion049","Port":38160},"Maximum Memory":1611635097,"Timestamp":1419923345710}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"35","Host":"centurion050","Port":57131},"Maximum Memory":1611635097,"Timestamp":1419923345738}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"33","Host":"centurion045","Port":34591},"Maximum Memory":1611635097,"Timestamp":1419923345741}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"25","Host":"centurion032","Port":52587},"Maximum Memory":1611635097,"Timestamp":1419923345756}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"22","Host":"centurion027","Port":52744},"Maximum Memory":1611635097,"Timestamp":1419923345757}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"31","Host":"centurion054","Port":59083},"Maximum Memory":1611635097,"Timestamp":1419923345762}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"14","Host":"centurion035","Port":46565},"Maximum Memory":1611635097,"Timestamp":1419923345770}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"centurion036","Port":33453},"Maximum Memory":1611635097,"Timestamp":1419923345785}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"39","Host":"centurion010","Port":55661},"Maximum Memory":1611635097,"Timestamp":1419923345787}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"11","Host":"centurion034","Port":52112},"Maximum Memory":1611635097,"Timestamp":1419923345795}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":"centurion044","Port":43470},"Maximum Memory":1611635097,"Timestamp":1419923345801}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"37","Host":"centurion047","Port":34901},"Maximum Memory":1611635097,"Timestamp":1419923345814}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"9","Host":"centurion042","Port":59695},"Maximum Memory":1611635097,"Timestamp":1419923345814}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"centurion013","Port":40581},"Maximum Memory":1611635097,"Timestamp":1419923345841}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"38","Host":"centurion022","Port":50531},"Maximum Memory":1611635097,"Timestamp":1419923345842}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"3","Host":"centurion038","Port":51797},"Maximum Memory":1611635097,"Timestamp":1419923345862}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"26","Host":"centurion015","Port":38913},"Maximum Memory":1611635097,"Timestamp":1419923345880}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"32","Host":"centurion003","Port":44327},"Maximum Memory":1611635097,"Timestamp":1419923345884}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"43","Host":"centurion055","Port":60654},"Maximum Memory":1611635097,"Timestamp":1419923345891}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"17","Host":"centurion031","Port":37460},"Maximum Memory":1611635097,"Timestamp":1419923345902}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"4","Host":"centurion048","Port":54827},"Maximum Memory":1611635097,"Timestamp":1419923345907}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"18","Host":"centurion025","Port":47281},"Maximum Memory":1611635097,"Timestamp":1419923345911}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"19","Host":"centurion004","Port":41341},"Maximum Memory":1611635097,"Timestamp":1419923345928}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"6","Host":"centurion039","Port":60798},"Maximum Memory":1611635097,"Timestamp":1419923345953}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"28","Host":"centurion033","Port":50628},"Maximum Memory":1611635097,"Timestamp":1419923345964}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"29","Host":"centurion014","Port":54925},"Maximum Memory":1611635097,"Timestamp":1419923345991}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"16","Host":"centurion030","Port":50804},"Maximum Memory":1611635097,"Timestamp":1419923345995}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"20","Host":"centurion043","Port":47619},"Maximum Memory":1611635097,"Timestamp":1419923345998}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"36","Host":"centurion029","Port":47637},"Maximum Memory":1611635097,"Timestamp":1419923346008}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"44","Host":"centurion007","Port":49029},"Maximum Memory":1611635097,"Timestamp":1419923346031}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"41","Host":"centurion021","Port":52722},"Maximum Memory":1611635097,"Timestamp":1419923346043}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"42","Host":"centurion041","Port":35809},"Maximum Memory":1611635097,"Timestamp":1419923346047}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"10","Host":"centurion018","Port":50875},"Maximum Memory":1611635097,"Timestamp":1419923346094}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"12","Host":"centurion026","Port":40239},"Maximum Memory":1611635097,"Timestamp":1419923346101}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"13","Host":"centurion052","Port":60831},"Maximum Memory":1611635097,"Timestamp":1419923346105}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"34","Host":"centurion024","Port":57521},"Maximum Memory":1611635097,"Timestamp":1419923346185}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"15","Host":"centurion006","Port":35154},"Maximum Memory":1611635097,"Timestamp":1419923346213}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"40","Host":"centurion019","Port":49252},"Maximum Memory":1611635097,"Timestamp":1419923346243}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"30","Host":"centurion053","Port":42745},"Maximum Memory":1611635097,"Timestamp":1419923346261}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"8","Host":"centurion056","Port":46390},"Maximum Memory":1611635097,"Timestamp":1419923346298}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"45","Host":"centurion016","Port":49434},"Maximum Memory":1611635097,"Timestamp":1419923346399}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"24","Host":"centurion011","Port":42059},"Maximum Memory":1611635097,"Timestamp":1419923346434}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"5","Host":"centurion008","Port":49398},"Maximum Memory":1611635097,"Timestamp":1419923346515}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"7","Host":"centurion009","Port":40575},"Maximum Memory":1611635097,"Timestamp":1419923346534}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"27","Host":"centurion012","Port":49841},"Maximum Memory":1611635097,"Timestamp":1419923346549}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"46","Host":"centurion020","Port":50802},"Maximum Memory":1611635097,"Timestamp":1419923346700}