Hi kunal Can you list all S3 issues PR, we may need to give 1.4.2 patch release. Because aaron plan to use carbondata in production this month.
To arron : First please you try master, see if can solve your problems. Regards Liang kunalkapoor wrote > Hi aaron, > Many issues like this have been identified in 1.4 version. Most of the > issues have been fixed in the master code and will be released in 1.5 > version. > Remaing fixes are in progress. > Can you try the same scenario in 1.5(master branch). > > Thanks > Kunal Kapoor > > On Mon, Sep 3, 2018, 5:57 AM aaron < > 949835961@ >> wrote: > >> *update the aws-java-sdk and hadoop-aws to below version, then >> authorization >> works. >> com.amazonaws:aws-java-sdk:1.10.75.1,org.apache.hadoop:hadoop-aws:2.7.3* >> >> *But we still can not load data, the exception is same. >> carbon.sql("LOAD DATA INPATH >> 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE >> test_s3_table")* >> >> 18/09/02 21:49:47 ERROR CarbonLoaderUtil: main Unable to unlock Table >> lock >> for tabledefault.test_s3_table during table status updation >> 18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main >> java.lang.ArrayIndexOutOfBoundsException >> at java.lang.System.arraycopy(Native Method) >> at >> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128) >> at >> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164) >> at >> >> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58) >> at java.io.DataOutputStream.write(DataOutputStream.java:107) >> at >> >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111) >> at >> >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93) >> at >> >> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276) >> at >> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96) >> at >> >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41) >> at >> >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446) >> at >> >> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263) >> at >> >> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92) >> at >> >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) >> at >> >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) >> at >> >> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67) >> at org.apache.spark.sql.Dataset. > <init> > (Dataset.scala:183) >> at >> >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107) >> at >> >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96) >> at >> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154) >> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94) >> at >> >> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :34) >> at >> >> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :39) >> at >> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :41) >> at >> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :43) >> at >> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :45) >> at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :47) >> at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :49) >> at $line25.$read$$iw$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :51) >> at $line25.$read$$iw$$iw$$iw$$iw. > <init> > ( > <console> > :53) >> at $line25.$read$$iw$$iw$$iw. > <init> > ( > <console> > :55) >> at $line25.$read$$iw$$iw. > <init> > ( > <console> > :57) >> at $line25.$read$$iw. > <init> > ( > <console> > :59) >> at $line25.$read. > <init> > ( > <console> > :61) >> at $line25.$read$. > <init> > ( > <console> > :65) >> at $line25.$read$. > <clinit> > ( > <console> > ) >> at $line25.$eval$.$print$lzycompute( > <console> > :7) >> at $line25.$eval$.$print( > <console> > :6) >> at $line25.$eval.$print( > <console> > ) >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> at >> >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) >> at >> >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >> at java.lang.reflect.Method.invoke(Method.java:498) >> at >> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) >> at >> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) >> at >> >> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) >> at >> >> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) >> at >> >> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) >> at >> >> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) >> at >> >> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) >> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) >> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) >> at >> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807) >> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) >> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) >> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) >> at >> >> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923) >> at >> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) >> at >> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) >> at >> >> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97) >> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) >> at org.apache.spark.repl.Main$.doMain(Main.scala:74) >> at org.apache.spark.repl.Main$.main(Main.scala:54) >> at org.apache.spark.repl.Main.main(Main.scala) >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> at >> >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) >> at >> >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >> at java.lang.reflect.Method.invoke(Method.java:498) >> at >> >> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775) >> at >> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) >> at >> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) >> at >> org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119) >> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) >> 18/09/02 21:49:47 AUDIT CarbonLoadDataCommand: >> [aaron.local][aaron][Thread-1]Dataload failure for default.test_s3_table. >> Please check the logs >> 18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main Got exception >> java.lang.ArrayIndexOutOfBoundsException when processing data. But this >> command does not support undo yet, skipping the undo part. >> java.lang.ArrayIndexOutOfBoundsException >> at java.lang.System.arraycopy(Native Method) >> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128) >> at >> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164) >> at >> >> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58) >> at java.io.DataOutputStream.write(DataOutputStream.java:107) >> at >> >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111) >> at >> >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93) >> at >> >> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276) >> at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96) >> at >> >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41) >> at >> >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437) >> at >> >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446) >> at >> >> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263) >> at >> >> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92) >> at >> >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) >> at >> >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) >> at >> >> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67) >> at org.apache.spark.sql.Dataset. > <init> > (Dataset.scala:183) >> at >> >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107) >> at >> >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96) >> at >> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154) >> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94) >> ... 52 elided >> >> aaron wrote >> > Hi dear community, could anybody please kindly tell me what happened? >> > >> > *Env*: >> > >> > 1.spark 2.2.1 + carbon1.4.1 >> > 2.spark.jars.packages >> > com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.2 >> > 3.spark.driver.extraClassPath >> > file:///usr/local/Cellar/apache-spark/2.2.1/lib/* >> > spark.executor.extraClassPath >> > file:///usr/local/Cellar/apache-spark/2.2.1/lib/* >> > lib folder include below jars >> > -rw-r--r--@ 1 aaron staff 52M Aug 29 20:50 >> > apache-carbondata-1.4.1-bin-spark2.2.1-hadoop2.7.2.jar >> > -rw-r--r-- 1 aaron staff 764K Aug 29 21:33 httpclient-4.5.4.jar >> > -rw-r--r-- 1 aaron staff 314K Aug 29 21:40 httpcore-4.4.jar >> > >> > >> > *Code*: >> > >> > import org.apache.spark.sql.SparkSession >> > import org.apache.spark.sql.CarbonSession._ >> > import org.apache.spark.sql.catalyst.util._ >> > import org.apache.carbondata.core.util.CarbonProperties >> > import org.apache.carbondata.core.constants.CarbonCommonConstants >> > >> CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE, >> > "HDFSLOCK") >> > val carbon = >> > >> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl", >> > >> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key", >> > "xxx").config("spark.hadoop.fs.s3a.secret.key", >> > >> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta") >> > >> > carbon.sql("CREATE TABLE IF NOT EXISTS test_s3_table(id string, name >> > string, >> > city string, age Int) STORED BY 'carbondata' LOCATION >> > 's3a://key:password@aaron-s3-poc/'") >> > carbon.sql("LOAD DATA INPATH >> > 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE >> > test_s3_table") >> > >> > *s3 files,* >> > >> > aws s3 ls s3://aaron-s3-poc/ --human --recursive >> > 2018-08-29 22:13:32 0 Bytes LockFiles/tablestatus.lock >> > 2018-08-29 21:41:36 616 Bytes Metadata/schema >> > >> > >> > *Issue 1,* when I create table, carbondata raise Exception >> > "com.amazonaws.AmazonClientException: Unable to load AWS credentials >> from >> > any provider in the chain" even if >> > a. I set related properties in spark-default.conf like >> > spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem >> > spark.hadoop.fs.s3a.awsAccessKeyId=xxx >> > spark.hadoop.fs.s3a.awsSecretAccessKey=xxx >> > spark.hadoop.fs.s3a.access.key=xxx >> > spark.hadoop.fs.s3a.secret.key=xxx >> > b.config in code >> > val carbon = >> > >> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl", >> > >> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key", >> > "xxx").config("spark.hadoop.fs.s3a.secret.key", >> > >> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta") >> > c. spark-submit conf >> > Finally I succeed when I put credentials in LOCATION >> > 's3a://key:password@aaron-s3-poc/'", But it's very strange. Who could >> tell >> > me why? >> > >> > >> > *Issue 2,* Load data failed >> > >> > scala> carbon.sql("LOAD DATA INPATH >> > 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE >> > test_s3_table") >> > 18/08/29 22:13:35 ERROR CarbonLoaderUtil: main Unable to unlock Table >> lock >> > for tabledefault.test_s3_table during table status updation >> > 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main >> > java.lang.ArrayIndexOutOfBoundsException >> > at java.lang.System.arraycopy(Native Method) >> > at >> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128) >> > at >> > >> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164) >> > at >> > >> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58) >> > at java.io.DataOutputStream.write(DataOutputStream.java:107) >> > at >> > >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111) >> > at >> > >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93) >> > at >> > >> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276) >> > at >> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96) >> > at >> > >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41) >> > at >> > >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446) >> > at >> > >> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263) >> > at >> > >> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92) >> > at >> > >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) >> > at >> > >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) >> > at >> > >> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67) >> > at org.apache.spark.sql.Dataset. >> > > <init> >> > (Dataset.scala:183) >> > at >> > >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107) >> > at >> > >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96) >> > at >> > >> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154) >> > at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94) >> > at >> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :36) >> > at >> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :41) >> > at >> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :43) >> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :45) >> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :47) >> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :49) >> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :51) >> > at $line21.$read$$iw$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :53) >> > at $line21.$read$$iw$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :55) >> > at $line21.$read$$iw$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :57) >> > at $line21.$read$$iw$$iw. >> > > <init> >> > ( >> > > <console> >> > :59) >> > at $line21.$read$$iw. >> > > <init> >> > ( >> > > <console> >> > :61) >> > at $line21.$read. >> > > <init> >> > ( >> > > <console> >> > :63) >> > at $line21.$read$. >> > > <init> >> > ( >> > > <console> >> > :67) >> > at $line21.$read$. >> > > <clinit> >> > ( >> > > <console> >> > ) >> > at $line21.$eval$.$print$lzycompute( >> > > <console> >> > :7) >> > at $line21.$eval$.$print( >> > > <console> >> > :6) >> > at $line21.$eval.$print( >> > > <console> >> > ) >> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> > at >> > >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) >> > at >> > >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >> > at java.lang.reflect.Method.invoke(Method.java:498) >> > at >> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) >> > at >> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) >> > at >> > >> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) >> > at >> > >> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) >> > at >> > >> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) >> > at >> > >> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) >> > at >> > >> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) >> > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) >> > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) >> > at >> > >> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807) >> > at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) >> > at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) >> > at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) >> > at >> > >> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923) >> > at >> > >> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) >> > at >> > >> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) >> > at >> > >> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97) >> > at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) >> > at org.apache.spark.repl.Main$.doMain(Main.scala:74) >> > at org.apache.spark.repl.Main$.main(Main.scala:54) >> > at org.apache.spark.repl.Main.main(Main.scala) >> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> > at >> > >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) >> > at >> > >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >> > at java.lang.reflect.Method.invoke(Method.java:498) >> > at >> > >> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775) >> > at >> > org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) >> > at >> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) >> > at >> org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119) >> > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) >> > 18/08/29 22:13:35 AUDIT CarbonLoadDataCommand: >> > [aaron.local][aaron][Thread-1]Dataload failure for >> default.test_s3_table. >> > Please check the logs >> > 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main Got exception >> > java.lang.ArrayIndexOutOfBoundsException when processing data. But this >> > command does not support undo yet, skipping the undo part. >> > java.lang.ArrayIndexOutOfBoundsException >> > at java.lang.System.arraycopy(Native Method) >> > at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128) >> > at >> > >> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164) >> > at >> > >> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58) >> > at java.io.DataOutputStream.write(DataOutputStream.java:107) >> > at >> > >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111) >> > at >> > >> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93) >> > at >> > >> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276) >> > at >> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96) >> > at >> > >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41) >> > at >> > >> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437) >> > at >> > >> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446) >> > at >> > >> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263) >> > at >> > >> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92) >> > at >> > >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) >> > at >> > >> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) >> > at >> > >> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67) >> > at org.apache.spark.sql.Dataset. >> > > <init> >> > (Dataset.scala:183) >> > at >> > >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107) >> > at >> > >> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96) >> > at >> > >> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154) >> > at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94) >> > ... 52 elided >> > >> > >> > Thanks >> > Aaron >> > >> > >> > >> > -- >> > Sent from: >> > >> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/ >> >> >> >> >> >> -- >> Sent from: >> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/ >> -- Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/