more interesting is if spark-shell started on master node (test01) then
parquetFile.saveAsParquetFile("tachyon://test01.zala:19998/parquet_tablex") 14/08/12 11:42:06 INFO : initialize(tachyon://... ... ... 14/08/12 11:42:06 INFO : File does not exist: tachyon://test01.zala:19998/parquet_tablex/_metadata 14/08/12 11:42:06 INFO : getWorkingDirectory: / 14/08/12 11:42:06 INFO : create(tachyon://test01.zala:19998/parquet_tablex/_metadata, rw-r--r--, true, 65536, 1, 33554432, null) 14/08/12 11:42:06 WARN : tachyon.home is not set. Using /mnt/tachyon_default_home as the default value. 14/08/12 11:42:06 INFO : Trying to get local worker host : test01.zala 14/08/12 11:42:06 ERROR : No local worker on test01.zala NoWorkerException(message:No local worker on test01.zala) at tachyon.thrift.MasterService$user_getWorker_result$user_getWorker_resultStandardScheme.read(MasterService.java:25675) at tachyon.thrift.MasterService$user_getWorker_result$user_getWorker_resultStandardScheme.read(MasterService.java:25652) at tachyon.thrift.MasterService$user_getWorker_result.read(MasterService.java:25591) at tachyon.org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78) at tachyon.thrift.MasterService$Client.recv_user_getWorker(MasterService.java:832) at tachyon.thrift.MasterService$Client.user_getWorker(MasterService.java:818) at tachyon.master.MasterClient.user_getWorker(MasterClient.java:648) at tachyon.worker.WorkerClient.connect(WorkerClient.java:199) at tachyon.worker.WorkerClient.mustConnect(WorkerClient.java:360) at tachyon.worker.WorkerClient.getUserUfsTempFolder(WorkerClient.java:298) at tachyon.client.TachyonFS.createAndGetUserUfsTempFolder(TachyonFS.java:270) at tachyon.client.FileOutStream.<init>(FileOutStream.java:72) at tachyon.client.TachyonFile.getOutStream(TachyonFile.java:207) at tachyon.hadoop.AbstractTFS.create(AbstractTFS.java:102) at tachyon.hadoop.TFS.create(TFS.java:24) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:887) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:784) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:773) at parquet.hadoop.ParquetFileWriter.writeMetadataFile(ParquetFileWriter.java:344) at org.apache.spark.sql.parquet.ParquetTypesConverter$.writeMetaData(ParquetTypes.scala:345) at org.apache.spark.sql.parquet.ParquetRelation$.createEmpty(ParquetRelation.scala:142) at org.apache.spark.sql.parquet.ParquetRelation$.create(ParquetRelation.scala:120) at org.apache.spark.sql.execution.SparkStrategies$ParquetOperations$.apply(SparkStrategies.scala:197) at org.apache.spark.sql.catalyst.planning.QueryPlanner$$anonfun$1.apply(QueryPlanner.scala:58) at org.apache.spark.sql.catalyst.planning.QueryPlanner$$anonfun$1.apply(QueryPlanner.scala:58) at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371) at org.apache.spark.sql.catalyst.planning.QueryPlanner.apply(QueryPlanner.scala:59) at org.apache.spark.sql.SQLContext$QueryExecution.sparkPlan$lzycompute(SQLContext.scala:399) at org.apache.spark.sql.SQLContext$QueryExecution.sparkPlan(SQLContext.scala:397) at org.apache.spark.sql.SQLContext$QueryExecution.executedPlan$lzycompute(SQLContext.scala:403) at org.apache.spark.sql.SQLContext$QueryExecution.executedPlan(SQLContext.scala:403) at org.apache.spark.sql.SQLContext$QueryExecution.toRdd$lzycompute(SQLContext.scala:406) at org.apache.spark.sql.SQLContext$QueryExecution.toRdd(SQLContext.scala:406) at org.apache.spark.sql.SchemaRDDLike$class.saveAsParquetFile(SchemaRDDLike.scala:77) at org.apache.spark.sql.SchemaRDD.saveAsParquetFile(SchemaRDD.scala:103) at $line12.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:17) at $line12.$read$$iwC$$iwC$$iwC.<init>(<console>:22) at $line12.$read$$iwC$$iwC.<init>(<console>:24) at $line12.$read$$iwC.<init>(<console>:26) at $line12.$read.<init>(<console>:28) at $line12.$read$.<init>(<console>:32) at $line12.$read$.<clinit>(<console>) at $line12.$eval$.<init>(<console>:7) at $line12.$eval$.<clinit>(<console>) at $line12.$eval.$print(<console>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:789) at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1062) at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:615) at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:646) at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:610) at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814) at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:859) at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:771) at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:616) at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:624) at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:629) at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:954) at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:902) at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:902) at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135) at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:902) at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:997) at org.apache.spark.repl.Main$.main(Main.scala:31) at org.apache.spark.repl.Main.main(Main.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:314) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:73) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 14/08/12 11:42:06 INFO : Trying to connect master @ test01.zala/10.10.114.120:19998 14/08/12 11:42:06 INFO : User registered at the master test01.zala/10.10.114.120:19998 got UserId 32233 14/08/12 11:42:06 ERROR : No local worker on test01.zala NoWorkerException(message:No local worker on test01.zala) at tachyon.thrift.MasterService$user_getWorker_result$user_getWorker_resultStandardScheme.read(MasterService.java:25675) at tachyon.thrift.MasterService$user_getWorker_result$user_getWorker_resultStandardScheme.read(MasterService.java:25652) at tachyon.thrift.MasterService$user_getWorker_result.read(MasterService.java:25591) at tachyon.org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78) at tachyon.thrift.MasterService$Client.recv_user_getWorker(MasterService.java:832) at tachyon.thrift.MasterService$Client.user_getWorker(MasterService.java:818) at tachyon.master.MasterClient.user_getWorker(MasterClient.java:648) at tachyon.worker.WorkerClient.connect(WorkerClient.java:199) at tachyon.worker.WorkerClient.mustConnect(WorkerClient.java:360) at tachyon.worker.WorkerClient.getUserUfsTempFolder(WorkerClient.java:298) at tachyon.client.TachyonFS.createAndGetUserUfsTempFolder(TachyonFS.java:270) at tachyon.client.FileOutStream.<init>(FileOutStream.java:72) at tachyon.client.TachyonFile.getOutStream(TachyonFile.java:207) at tachyon.hadoop.AbstractTFS.create(AbstractTFS.java:102) at tachyon.hadoop.TFS.create(TFS.java:24) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:887) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:784) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:773) at parquet.hadoop.ParquetFileWriter.writeMetadataFile(ParquetFileWriter.java:344) at org.apache.spark.sql.parquet.ParquetTypesConverter$.writeMetaData(ParquetTypes.scala:345) at org.apache.spark.sql.parquet.ParquetRelation$.createEmpty(ParquetRelation.scala:142) at org.apache.spark.sql.parquet.ParquetRelation$.create(ParquetRelation.scala:120) at org.apache.spark.sql.execution.SparkStrategies$ParquetOperations$.apply(SparkStrategies.scala:197) at org.apache.spark.sql.catalyst.planning.QueryPlanner$$anonfun$1.apply(QueryPlanner.scala:58) at org.apache.spark.sql.catalyst.planning.QueryPlanner$$anonfun$1.apply(QueryPlanner.scala:58) at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371) at org.apache.spark.sql.catalyst.planning.QueryPlanner.apply(QueryPlanner.scala:59) at org.apache.spark.sql.SQLContext$QueryExecution.sparkPlan$lzycompute(SQLContext.scala:399) at org.apache.spark.sql.SQLContext$QueryExecution.sparkPlan(SQLContext.scala:397) at org.apache.spark.sql.SQLContext$QueryExecution.executedPlan$lzycompute(SQLContext.scala:403) at org.apache.spark.sql.SQLContext$QueryExecution.executedPlan(SQLContext.scala:403) at org.apache.spark.sql.SQLContext$QueryExecution.toRdd$lzycompute(SQLContext.scala:406) at org.apache.spark.sql.SQLContext$QueryExecution.toRdd(SQLContext.scala:406) at org.apache.spark.sql.SchemaRDDLike$class.saveAsParquetFile(SchemaRDDLike.scala:77) at org.apache.spark.sql.SchemaRDD.saveAsParquetFile(SchemaRDD.scala:103) at $line12.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:17) at $line12.$read$$iwC$$iwC$$iwC.<init>(<console>:22) at $line12.$read$$iwC$$iwC.<init>(<console>:24) at $line12.$read$$iwC.<init>(<console>:26) at $line12.$read.<init>(<console>:28) at $line12.$read$.<init>(<console>:32) at $line12.$read$.<clinit>(<console>) at $line12.$eval$.<init>(<console>:7) at $line12.$eval$.<clinit>(<console>) at $line12.$eval.$print(<console>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:789) at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1062) at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:615) at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:646) at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:610) at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:814) at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:859) at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:771) at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:616) at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:624) at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:629) at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:954) at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:902) at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:902) at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135) at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:902) at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:997) at org.apache.spark.repl.Main$.main(Main.scala:31) at org.apache.spark.repl.Main.main(Main.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:314) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:73) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) infinite error message like 14/08/12 11:42:06 INFO : Trying to connect master @ test01.zala/10.10.114.120:19998 14/08/12 11:42:06 INFO : User registered at the master test01.zala/10.10.114.120:19998 got UserId 32233 14/08/12 11:42:06 ERROR : No local worker on test01.zala NoWorkerException(message:No local worker on test01.zala) always "got UserId XXX" and then "No local worker on test01.zala" save text files works fine with spark-shell on both master node and worker node -- View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/share-reuse-off-heap-persisted-tachyon-RDD-in-SparkContext-or-saveAsParquetFile-on-tachyon-in-SQLCont-tp11897p11978.html Sent from the Apache Spark User List mailing list archive at Nabble.com. --------------------------------------------------------------------- To unsubscribe, e-mail: user-unsubscr...@spark.apache.org For additional commands, e-mail: user-h...@spark.apache.org