[ https://issues.apache.org/jira/browse/SPARK-9539?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Andrey Zimovnov updated SPARK-9539: ----------------------------------- Description: Example code in Python: {code:python} for i in range(20): print i conf = SparkConf().setAppName("test") sc = SparkContext(conf=conf) hivec = HiveContext(sc) hivec.sql("select id from details_info limit 1").show() sc.stop() del hivec del sc {code} Jstat output: {noformat} S0C S1C S0U S1U EC EU OC OU PC PU YGC YGCT FGC FGCT GCT 196608,0 196608,0 97566,2 0,0 1179648,0 542150,0 3145728,0 120,0 154112,0 153613,2 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 679041,7 3145728,0 120,0 164352,0 164183,3 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 907928,4 3145728,0 120,0 164352,0 164200,3 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 912132,7 3145728,0 120,0 164352,0 164200,5 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 913741,5 3145728,0 120,0 164352,0 164200,8 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 929458,6 3145728,0 120,0 164352,0 164206,0 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 1003138,1 3145728,0 120,0 168960,0 168646,0 4 0,434 0 0,000 0,434 131584,0 196608,0 0,0 109725,6 1179648,0 0,0 3145728,0 128,0 175104,0 174802,1 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 152654,9 3145728,0 128,0 175104,0 174803,3 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 158586,1 3145728,0 128,0 175104,0 174803,3 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 160659,8 3145728,0 128,0 175104,0 174805,7 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 181935,2 3145728,0 128,0 175104,0 174819,7 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 283389,1 3145728,0 128,0 185856,0 185371,0 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 342596,4 3145728,0 128,0 185856,0 185379,3 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 547634,7 3145728,0 128,0 185856,0 185385,8 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 555930,9 3145728,0 128,0 185856,0 185385,8 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 557888,6 3145728,0 128,0 185856,0 185386,0 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 573907,5 3145728,0 128,0 185856,0 185397,5 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 637955,0 3145728,0 128,0 189952,0 189533,1 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 895866,1 3145728,0 128,0 196096,0 195968,5 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 948046,5 3145728,0 128,0 196096,0 195969,4 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 952427,2 3145728,0 128,0 196096,0 195969,4 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 957977,5 3145728,0 128,0 196096,0 195973,4 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 977811,1 3145728,0 128,0 196096,0 195977,7 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 1118722,0 3145728,0 128,0 206848,0 206539,0 5 0,591 0 0,000 0,591 131584,0 144384,0 118692,5 0,0 1284096,0 183470,8 3145728,0 136,0 206848,0 206543,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 189718,5 3145728,0 136,0 206848,0 206543,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 192165,0 3145728,0 136,0 206848,0 206543,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 199848,4 3145728,0 136,0 206848,0 206546,9 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 219687,6 3145728,0 136,0 206848,0 206552,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 358272,4 3145728,0 136,0 217600,0 217100,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 573543,6 3145728,0 136,0 217600,0 217109,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 593569,9 3145728,0 136,0 217600,0 217109,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 593576,3 3145728,0 136,0 217600,0 217109,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 608832,1 3145728,0 136,0 217600,0 217114,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 649072,6 3145728,0 136,0 218624,0 218234,7 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 774293,0 3145728,0 136,0 227840,0 227734,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 971194,9 3145728,0 136,0 227840,0 227739,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 992291,6 3145728,0 136,0 227840,0 227739,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 992297,7 3145728,0 136,0 227840,0 227739,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 1002356,2 3145728,0 136,0 227840,0 227745,1 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 1020944,8 3145728,0 136,0 227840,0 227754,0 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 1169159,0 3145728,0 136,0 238592,0 238332,9 6 0,773 0 0,000 0,773 147456,0 109056,0 0,0 108867,6 1284096,0 110659,1 3145728,0 19167,5 238592,0 238479,1 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 116954,2 3145728,0 19167,5 238592,0 238491,0 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 116960,6 3145728,0 19167,5 238592,0 238491,0 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 132244,5 3145728,0 19167,5 238592,0 238491,9 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 218816,2 3145728,0 19167,5 245760,0 244549,2 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 467076,0 3145728,0 19167,5 249344,0 249189,3 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 508023,8 3145728,0 19167,5 249344,0 249189,8 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 511485,1 3145728,0 19167,5 249344,0 249191,3 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 513094,4 3145728,0 19167,5 249344,0 249191,9 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 527507,0 3145728,0 19167,5 249344,0 249197,0 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 636565,3 3145728,0 19167,5 260096,0 259705,2 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 819041,0 3145728,0 19167,5 260096,0 259749,7 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 917166,3 3145728,0 19167,5 260096,0 259749,7 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 919595,7 3145728,0 19167,5 260096,0 259749,7 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 925158,0 3145728,0 19167,5 260096,0 259750,1 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 946351,0 3145728,0 19167,5 260096,0 259751,1 7 0,942 0 0,000 0,942 151552,0 1024,0 0,0 924,7 1284608,0 0,0 3145728,0 119860,5 262144,0 262143,9 9 1,038 2 2,021 3,059 148480,0 153088,0 0,0 64,0 1262080,0 0,0 3145728,0 84562,9 262144,0 262143,9 17 1,167 10 6,988 8,155 99840,0 93696,0 160,0 0,0 1379328,0 0,0 3145728,0 84035,5 262144,0 262143,6 26 1,296 19 12,022 13,318 50688,0 54272,0 0,0 224,0 1460224,0 0,0 3145728,0 83958,1 262144,0 262143,8 35 1,474 28 16,642 18,116 23552,0 25088,0 0,0 96,0 1520640,0 0,0 3145728,0 84007,6 262144,0 262144,0 45 1,638 38 21,492 23,130 {noformat} It looks like it hits -XX:MaxPermSize=256m for the driver and dies with OOM: Exception: ("You must build Spark with Hive. Export 'SPARK_HIVE=true' and run build/sbt assembly", Py4JJavaError(u'An error occurred while calling None.org.apache.spark.sql.hive.HiveContext.\n', JavaObject id=o753)) was: Example code in Python: for i in range(20): print i conf = SparkConf().setAppName("test") sc = SparkContext(conf=conf) hivec = HiveContext(sc) hivec.sql("select id from details_info limit 1").show() sc.stop() del hivec del sc Jstat output: S0C S1C S0U S1U EC EU OC OU PC PU YGC YGCT FGC FGCT GCT 196608,0 196608,0 97566,2 0,0 1179648,0 542150,0 3145728,0 120,0 154112,0 153613,2 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 679041,7 3145728,0 120,0 164352,0 164183,3 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 907928,4 3145728,0 120,0 164352,0 164200,3 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 912132,7 3145728,0 120,0 164352,0 164200,5 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 913741,5 3145728,0 120,0 164352,0 164200,8 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 929458,6 3145728,0 120,0 164352,0 164206,0 4 0,434 0 0,000 0,434 196608,0 196608,0 97566,2 0,0 1179648,0 1003138,1 3145728,0 120,0 168960,0 168646,0 4 0,434 0 0,000 0,434 131584,0 196608,0 0,0 109725,6 1179648,0 0,0 3145728,0 128,0 175104,0 174802,1 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 152654,9 3145728,0 128,0 175104,0 174803,3 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 158586,1 3145728,0 128,0 175104,0 174803,3 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 160659,8 3145728,0 128,0 175104,0 174805,7 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 181935,2 3145728,0 128,0 175104,0 174819,7 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 283389,1 3145728,0 128,0 185856,0 185371,0 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 342596,4 3145728,0 128,0 185856,0 185379,3 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 547634,7 3145728,0 128,0 185856,0 185385,8 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 555930,9 3145728,0 128,0 185856,0 185385,8 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 557888,6 3145728,0 128,0 185856,0 185386,0 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 573907,5 3145728,0 128,0 185856,0 185397,5 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 637955,0 3145728,0 128,0 189952,0 189533,1 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 895866,1 3145728,0 128,0 196096,0 195968,5 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 948046,5 3145728,0 128,0 196096,0 195969,4 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 952427,2 3145728,0 128,0 196096,0 195969,4 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 957977,5 3145728,0 128,0 196096,0 195973,4 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 977811,1 3145728,0 128,0 196096,0 195977,7 5 0,591 0 0,000 0,591 131584,0 196608,0 0,0 109725,6 1179648,0 1118722,0 3145728,0 128,0 206848,0 206539,0 5 0,591 0 0,000 0,591 131584,0 144384,0 118692,5 0,0 1284096,0 183470,8 3145728,0 136,0 206848,0 206543,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 189718,5 3145728,0 136,0 206848,0 206543,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 192165,0 3145728,0 136,0 206848,0 206543,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 199848,4 3145728,0 136,0 206848,0 206546,9 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 219687,6 3145728,0 136,0 206848,0 206552,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 358272,4 3145728,0 136,0 217600,0 217100,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 573543,6 3145728,0 136,0 217600,0 217109,4 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 593569,9 3145728,0 136,0 217600,0 217109,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 593576,3 3145728,0 136,0 217600,0 217109,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 608832,1 3145728,0 136,0 217600,0 217114,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 649072,6 3145728,0 136,0 218624,0 218234,7 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 774293,0 3145728,0 136,0 227840,0 227734,6 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 971194,9 3145728,0 136,0 227840,0 227739,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 992291,6 3145728,0 136,0 227840,0 227739,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 992297,7 3145728,0 136,0 227840,0 227739,2 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 1002356,2 3145728,0 136,0 227840,0 227745,1 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 1020944,8 3145728,0 136,0 227840,0 227754,0 6 0,773 0 0,000 0,773 131584,0 144384,0 118692,5 0,0 1284096,0 1169159,0 3145728,0 136,0 238592,0 238332,9 6 0,773 0 0,000 0,773 147456,0 109056,0 0,0 108867,6 1284096,0 110659,1 3145728,0 19167,5 238592,0 238479,1 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 116954,2 3145728,0 19167,5 238592,0 238491,0 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 116960,6 3145728,0 19167,5 238592,0 238491,0 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 132244,5 3145728,0 19167,5 238592,0 238491,9 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 218816,2 3145728,0 19167,5 245760,0 244549,2 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 467076,0 3145728,0 19167,5 249344,0 249189,3 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 508023,8 3145728,0 19167,5 249344,0 249189,8 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 511485,1 3145728,0 19167,5 249344,0 249191,3 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 513094,4 3145728,0 19167,5 249344,0 249191,9 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 527507,0 3145728,0 19167,5 249344,0 249197,0 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 636565,3 3145728,0 19167,5 260096,0 259705,2 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 819041,0 3145728,0 19167,5 260096,0 259749,7 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 917166,3 3145728,0 19167,5 260096,0 259749,7 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 919595,7 3145728,0 19167,5 260096,0 259749,7 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 925158,0 3145728,0 19167,5 260096,0 259750,1 7 0,942 0 0,000 0,942 147456,0 109056,0 0,0 108867,6 1284096,0 946351,0 3145728,0 19167,5 260096,0 259751,1 7 0,942 0 0,000 0,942 151552,0 1024,0 0,0 924,7 1284608,0 0,0 3145728,0 119860,5 262144,0 262143,9 9 1,038 2 2,021 3,059 148480,0 153088,0 0,0 64,0 1262080,0 0,0 3145728,0 84562,9 262144,0 262143,9 17 1,167 10 6,988 8,155 99840,0 93696,0 160,0 0,0 1379328,0 0,0 3145728,0 84035,5 262144,0 262143,6 26 1,296 19 12,022 13,318 50688,0 54272,0 0,0 224,0 1460224,0 0,0 3145728,0 83958,1 262144,0 262143,8 35 1,474 28 16,642 18,116 23552,0 25088,0 0,0 96,0 1520640,0 0,0 3145728,0 84007,6 262144,0 262144,0 45 1,638 38 21,492 23,130 It looks like it hits -XX:MaxPermSize=256m for the driver and dies with OOM: Exception: ("You must build Spark with Hive. Export 'SPARK_HIVE=true' and run build/sbt assembly", Py4JJavaError(u'An error occurred while calling None.org.apache.spark.sql.hive.HiveContext.\n', JavaObject id=o753)) > Repeated sc.close() in PySpark causes JVM memory leak > ----------------------------------------------------- > > Key: SPARK-9539 > URL: https://issues.apache.org/jira/browse/SPARK-9539 > Project: Spark > Issue Type: Bug > Components: PySpark > Affects Versions: 1.4.1 > Reporter: Andrey Zimovnov > Priority: Minor > > Example code in Python: > {code:python} > for i in range(20): > print i > conf = SparkConf().setAppName("test") > sc = SparkContext(conf=conf) > hivec = HiveContext(sc) > hivec.sql("select id from details_info limit 1").show() > sc.stop() > del hivec > del sc > {code} > Jstat output: > {noformat} > S0C S1C S0U S1U EC EU OC OU PC > PU YGC YGCT FGC FGCT GCT > 196608,0 196608,0 97566,2 0,0 1179648,0 542150,0 3145728,0 120,0 > 154112,0 153613,2 4 0,434 0 0,000 0,434 > 196608,0 196608,0 97566,2 0,0 1179648,0 679041,7 3145728,0 120,0 > 164352,0 164183,3 4 0,434 0 0,000 0,434 > 196608,0 196608,0 97566,2 0,0 1179648,0 907928,4 3145728,0 120,0 > 164352,0 164200,3 4 0,434 0 0,000 0,434 > 196608,0 196608,0 97566,2 0,0 1179648,0 912132,7 3145728,0 120,0 > 164352,0 164200,5 4 0,434 0 0,000 0,434 > 196608,0 196608,0 97566,2 0,0 1179648,0 913741,5 3145728,0 120,0 > 164352,0 164200,8 4 0,434 0 0,000 0,434 > 196608,0 196608,0 97566,2 0,0 1179648,0 929458,6 3145728,0 120,0 > 164352,0 164206,0 4 0,434 0 0,000 0,434 > 196608,0 196608,0 97566,2 0,0 1179648,0 1003138,1 3145728,0 120,0 > 168960,0 168646,0 4 0,434 0 0,000 0,434 > 131584,0 196608,0 0,0 109725,6 1179648,0 0,0 3145728,0 128,0 > 175104,0 174802,1 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 152654,9 3145728,0 128,0 > 175104,0 174803,3 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 158586,1 3145728,0 128,0 > 175104,0 174803,3 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 160659,8 3145728,0 128,0 > 175104,0 174805,7 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 181935,2 3145728,0 128,0 > 175104,0 174819,7 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 283389,1 3145728,0 128,0 > 185856,0 185371,0 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 342596,4 3145728,0 128,0 > 185856,0 185379,3 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 547634,7 3145728,0 128,0 > 185856,0 185385,8 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 555930,9 3145728,0 128,0 > 185856,0 185385,8 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 557888,6 3145728,0 128,0 > 185856,0 185386,0 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 573907,5 3145728,0 128,0 > 185856,0 185397,5 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 637955,0 3145728,0 128,0 > 189952,0 189533,1 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 895866,1 3145728,0 128,0 > 196096,0 195968,5 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 948046,5 3145728,0 128,0 > 196096,0 195969,4 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 952427,2 3145728,0 128,0 > 196096,0 195969,4 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 957977,5 3145728,0 128,0 > 196096,0 195973,4 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 977811,1 3145728,0 128,0 > 196096,0 195977,7 5 0,591 0 0,000 0,591 > 131584,0 196608,0 0,0 109725,6 1179648,0 1118722,0 3145728,0 128,0 > 206848,0 206539,0 5 0,591 0 0,000 0,591 > 131584,0 144384,0 118692,5 0,0 1284096,0 183470,8 3145728,0 136,0 > 206848,0 206543,4 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 189718,5 3145728,0 136,0 > 206848,0 206543,4 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 192165,0 3145728,0 136,0 > 206848,0 206543,4 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 199848,4 3145728,0 136,0 > 206848,0 206546,9 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 219687,6 3145728,0 136,0 > 206848,0 206552,2 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 358272,4 3145728,0 136,0 > 217600,0 217100,4 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 573543,6 3145728,0 136,0 > 217600,0 217109,4 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 593569,9 3145728,0 136,0 > 217600,0 217109,6 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 593576,3 3145728,0 136,0 > 217600,0 217109,6 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 608832,1 3145728,0 136,0 > 217600,0 217114,6 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 649072,6 3145728,0 136,0 > 218624,0 218234,7 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 774293,0 3145728,0 136,0 > 227840,0 227734,6 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 971194,9 3145728,0 136,0 > 227840,0 227739,2 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 992291,6 3145728,0 136,0 > 227840,0 227739,2 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 992297,7 3145728,0 136,0 > 227840,0 227739,2 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 1002356,2 3145728,0 136,0 > 227840,0 227745,1 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 1020944,8 3145728,0 136,0 > 227840,0 227754,0 6 0,773 0 0,000 0,773 > 131584,0 144384,0 118692,5 0,0 1284096,0 1169159,0 3145728,0 136,0 > 238592,0 238332,9 6 0,773 0 0,000 0,773 > 147456,0 109056,0 0,0 108867,6 1284096,0 110659,1 3145728,0 19167,5 > 238592,0 238479,1 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 116954,2 3145728,0 19167,5 > 238592,0 238491,0 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 116960,6 3145728,0 19167,5 > 238592,0 238491,0 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 132244,5 3145728,0 19167,5 > 238592,0 238491,9 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 218816,2 3145728,0 19167,5 > 245760,0 244549,2 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 467076,0 3145728,0 19167,5 > 249344,0 249189,3 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 508023,8 3145728,0 19167,5 > 249344,0 249189,8 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 511485,1 3145728,0 19167,5 > 249344,0 249191,3 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 513094,4 3145728,0 19167,5 > 249344,0 249191,9 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 527507,0 3145728,0 19167,5 > 249344,0 249197,0 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 636565,3 3145728,0 19167,5 > 260096,0 259705,2 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 819041,0 3145728,0 19167,5 > 260096,0 259749,7 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 917166,3 3145728,0 19167,5 > 260096,0 259749,7 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 919595,7 3145728,0 19167,5 > 260096,0 259749,7 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 925158,0 3145728,0 19167,5 > 260096,0 259750,1 7 0,942 0 0,000 0,942 > 147456,0 109056,0 0,0 108867,6 1284096,0 946351,0 3145728,0 19167,5 > 260096,0 259751,1 7 0,942 0 0,000 0,942 > 151552,0 1024,0 0,0 924,7 1284608,0 0,0 3145728,0 119860,5 > 262144,0 262143,9 9 1,038 2 2,021 3,059 > 148480,0 153088,0 0,0 64,0 1262080,0 0,0 3145728,0 84562,9 > 262144,0 262143,9 17 1,167 10 6,988 8,155 > 99840,0 93696,0 160,0 0,0 1379328,0 0,0 3145728,0 84035,5 > 262144,0 262143,6 26 1,296 19 12,022 13,318 > 50688,0 54272,0 0,0 224,0 1460224,0 0,0 3145728,0 83958,1 > 262144,0 262143,8 35 1,474 28 16,642 18,116 > 23552,0 25088,0 0,0 96,0 1520640,0 0,0 3145728,0 84007,6 > 262144,0 262144,0 45 1,638 38 21,492 23,130 > {noformat} > It looks like it hits -XX:MaxPermSize=256m for the driver and dies with OOM: > Exception: ("You must build Spark with Hive. Export 'SPARK_HIVE=true' and run > build/sbt assembly", Py4JJavaError(u'An error occurred while calling > None.org.apache.spark.sql.hive.HiveContext.\n', JavaObject id=o753)) -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org