[jira] [Updated] (KYLIN-3537) Use Spark to build Cube on Yarn faild at Setp8 on HDP3.0
[ https://issues.apache.org/jira/browse/KYLIN-3537?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3537: - Attachment: KYLIN-3537.master.001.patch > Use Spark to build Cube on Yarn faild at Setp8 on HDP3.0 > > > Key: KYLIN-3537 > URL: https://issues.apache.org/jira/browse/KYLIN-3537 > Project: Kylin > Issue Type: Bug > Environment: HDP3.0 >Reporter: Lijun Cao >Assignee: Lijun Cao >Priority: Major > Attachments: KYLIN-3537.master.001.patch > > > *The log on Yarn:* > 18/09/05 03:34:00 INFO scheduler.DAGScheduler: Job 0 failed: > saveAsNewAPIHadoopDataset at SparkCubeHFile.java:227, took 13.734642 s > 18/09/05 03:34:00 ERROR yarn.ApplicationMaster: User class threw exception: > java.lang.RuntimeException: error execute > org.apache.kylin.storage.hbase.steps.SparkCubeHFile > java.lang.RuntimeException: error execute > org.apache.kylin.storage.hbase.steps.SparkCubeHFile > at > org.apache.kylin.common.util.AbstractApplication.execute(AbstractApplication.java:42) > at org.apache.kylin.common.util.SparkEntry.main(SparkEntry.java:44) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:636) > Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: > Task 0 in stage 1.0 failed 4 times, most recent failure: Lost task 0.3 in > stage 1.0 (TID 14, ignite02.com, executor 2): java.lang.NoClassDefFoundError: > org/apache/hadoop/hbase/io/hfile/HFileWriterImpl > at > org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.createRecordWriter(HFileOutputFormat2.java:209) > at > org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.getRecordWriter(HFileOutputFormat2.java:181) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1$$anonfun$12.apply(PairRDDFunctions.scala:1119) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1$$anonfun$12.apply(PairRDDFunctions.scala:1102) > at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) > at org.apache.spark.scheduler.Task.run(Task.scala:99) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Driver stacktrace: > at > org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at > org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > at scala.Option.foreach(Option.scala:257) > at > org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594) > at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) > at > org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1928) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1941) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1961) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1158) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply(PairRDDFunctions.scala:1085) > at > org.apache.spark.rdd.PairRDDFunct
[jira] [Commented] (KYLIN-3537) Use Spark to build Cube on Yarn faild at Setp8 on HDP3.0
[ https://issues.apache.org/jira/browse/KYLIN-3537?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16773182#comment-16773182 ] Alexander commented on KYLIN-3537: -- Added dependicies and test it on my HDP 3.0.1 env. > Use Spark to build Cube on Yarn faild at Setp8 on HDP3.0 > > > Key: KYLIN-3537 > URL: https://issues.apache.org/jira/browse/KYLIN-3537 > Project: Kylin > Issue Type: Bug > Environment: HDP3.0 >Reporter: Lijun Cao >Assignee: Lijun Cao >Priority: Major > Attachments: KYLIN-3537.master.001.patch > > > *The log on Yarn:* > 18/09/05 03:34:00 INFO scheduler.DAGScheduler: Job 0 failed: > saveAsNewAPIHadoopDataset at SparkCubeHFile.java:227, took 13.734642 s > 18/09/05 03:34:00 ERROR yarn.ApplicationMaster: User class threw exception: > java.lang.RuntimeException: error execute > org.apache.kylin.storage.hbase.steps.SparkCubeHFile > java.lang.RuntimeException: error execute > org.apache.kylin.storage.hbase.steps.SparkCubeHFile > at > org.apache.kylin.common.util.AbstractApplication.execute(AbstractApplication.java:42) > at org.apache.kylin.common.util.SparkEntry.main(SparkEntry.java:44) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:636) > Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: > Task 0 in stage 1.0 failed 4 times, most recent failure: Lost task 0.3 in > stage 1.0 (TID 14, ignite02.com, executor 2): java.lang.NoClassDefFoundError: > org/apache/hadoop/hbase/io/hfile/HFileWriterImpl > at > org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.createRecordWriter(HFileOutputFormat2.java:209) > at > org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.getRecordWriter(HFileOutputFormat2.java:181) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1$$anonfun$12.apply(PairRDDFunctions.scala:1119) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1$$anonfun$12.apply(PairRDDFunctions.scala:1102) > at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) > at org.apache.spark.scheduler.Task.run(Task.scala:99) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Driver stacktrace: > at > org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at > org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > at scala.Option.foreach(Option.scala:257) > at > org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594) > at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) > at > org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1928) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1941) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1961) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1158) > at > org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsNewAPIHadoopDataset$1.apply(PairRDDFunc
[jira] [Created] (KYLIN-3824) Spark - Extract Fact Table Distinct Columns step causes java.lang.OutOfMemoryError: Java heap space
Alexander created KYLIN-3824: Summary: Spark - Extract Fact Table Distinct Columns step causes java.lang.OutOfMemoryError: Java heap space Key: KYLIN-3824 URL: https://issues.apache.org/jira/browse/KYLIN-3824 Project: Kylin Issue Type: Bug Components: Spark Engine Affects Versions: v2.6.1 Environment: CentOS 7 3 workers and 1 master. 4 cpu, 16GB RAM each Reporter: Alexander Try to build huge cube on weak envirment. Environment: Cluster with 3 nodes. Max AM container size - 5GB. kylin_intermediate table ~500 files of size started from 4kb up to 300mb. When spark job executor take file larger than ~70MB on step mapPartitionsToPair (194) it got exception: 2019-02-21 20:29:40 ERROR SparkUncaughtExceptionHandler:91 - [Container in shutdown] Uncaught exception in thread Thread[Executor task launch worker for task 1,5,main] java.lang.OutOfMemoryError: Java heap space at java.util.Arrays.copyOfRange(Arrays.java:3664) at java.lang.String.(String.java:207) at java.lang.String.substring(String.java:1969) at java.lang.String.split(String.java:2353) at java.lang.String.split(String.java:2422) at org.apache.kylin.engine.spark.SparkUtil$1.call(SparkUtil.java:164) at org.apache.kylin.engine.spark.SparkUtil$1.call(SparkUtil.java:160) at org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1.apply(JavaPairRDD.scala:1040) at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) at scala.collection.convert.Wrappers$IteratorWrapper.next(Wrappers.scala:31) at com.google.common.collect.Lists.newArrayList(Lists.java:145) at org.apache.kylin.engine.spark.SparkFactDistinct$FlatOutputFucntion.call(SparkFactDistinct.java:313) at org.apache.kylin.engine.spark.SparkFactDistinct$FlatOutputFucntion.call(SparkFactDistinct.java:239) at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:186) at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:186) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) at org.apache.spark.scheduler.Task.run(Task.scala:109) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (KYLIN-3824) Spark - Extract Fact Table Distinct Columns step causes java.lang.OutOfMemoryError: Java heap space
[ https://issues.apache.org/jira/browse/KYLIN-3824?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3824: - Attachment: KYLIN-3824.master.001.patch > Spark - Extract Fact Table Distinct Columns step causes > java.lang.OutOfMemoryError: Java heap space > --- > > Key: KYLIN-3824 > URL: https://issues.apache.org/jira/browse/KYLIN-3824 > Project: Kylin > Issue Type: Bug > Components: Spark Engine >Affects Versions: v2.6.1 > Environment: CentOS 7 > 3 workers and 1 master. > 4 cpu, 16GB RAM each >Reporter: Alexander >Priority: Major > Attachments: KYLIN-3824.master.001.patch > > > Try to build huge cube on weak envirment. > Environment: > Cluster with 3 nodes. > Max AM container size - 5GB. > > kylin_intermediate table ~500 files of size started from 4kb up to 300mb. > > When spark job executor take file larger than ~70MB on step > mapPartitionsToPair (194) it got exception: > 2019-02-21 20:29:40 ERROR SparkUncaughtExceptionHandler:91 - [Container in > shutdown] Uncaught exception in thread Thread[Executor task launch worker for > task 1,5,main] > java.lang.OutOfMemoryError: Java heap space > at java.util.Arrays.copyOfRange(Arrays.java:3664) > at java.lang.String.(String.java:207) > at java.lang.String.substring(String.java:1969) > at java.lang.String.split(String.java:2353) > at java.lang.String.split(String.java:2422) > at org.apache.kylin.engine.spark.SparkUtil$1.call(SparkUtil.java:164) > at org.apache.kylin.engine.spark.SparkUtil$1.call(SparkUtil.java:160) > at > org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1.apply(JavaPairRDD.scala:1040) > at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) > at scala.collection.convert.Wrappers$IteratorWrapper.next(Wrappers.scala:31) > at com.google.common.collect.Lists.newArrayList(Lists.java:145) > at > org.apache.kylin.engine.spark.SparkFactDistinct$FlatOutputFucntion.call(SparkFactDistinct.java:313) > at > org.apache.kylin.engine.spark.SparkFactDistinct$FlatOutputFucntion.call(SparkFactDistinct.java:239) > at > org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:186) > at > org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:186) > at > org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) > at > org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) > at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) > at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) > at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) > at org.apache.spark.scheduler.Task.run(Task.scala:109) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > > > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (KYLIN-3824) Spark - Extract Fact Table Distinct Columns step causes java.lang.OutOfMemoryError: Java heap space
[ https://issues.apache.org/jira/browse/KYLIN-3824?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander resolved KYLIN-3824. -- Resolution: Fixed Fix Version/s: (was: v2.6.1) v3.0.0 > Spark - Extract Fact Table Distinct Columns step causes > java.lang.OutOfMemoryError: Java heap space > --- > > Key: KYLIN-3824 > URL: https://issues.apache.org/jira/browse/KYLIN-3824 > Project: Kylin > Issue Type: Bug > Components: Spark Engine >Affects Versions: v2.5.0, v2.6.0, v2.5.1, v2.5.2 > Environment: CentOS 7 > 3 workers and 1 master. > 4 cpu, 16GB RAM each >Reporter: Alexander >Assignee: Alexander >Priority: Major > Fix For: v3.0.0 > > Attachments: KYLIN-3824.master.001.patch > > > Try to build huge cube on weak envirment. > Environment: > Cluster with 3 nodes. > Max AM container size - 5GB. > > kylin_intermediate table ~500 files of size started from 4kb up to 300mb. > > When spark job executor take file larger than ~70MB on step > mapPartitionsToPair (194) it got exception: > 2019-02-21 20:29:40 ERROR SparkUncaughtExceptionHandler:91 - [Container in > shutdown] Uncaught exception in thread Thread[Executor task launch worker for > task 1,5,main] > java.lang.OutOfMemoryError: Java heap space > at java.util.Arrays.copyOfRange(Arrays.java:3664) > at java.lang.String.(String.java:207) > at java.lang.String.substring(String.java:1969) > at java.lang.String.split(String.java:2353) > at java.lang.String.split(String.java:2422) > at org.apache.kylin.engine.spark.SparkUtil$1.call(SparkUtil.java:164) > at org.apache.kylin.engine.spark.SparkUtil$1.call(SparkUtil.java:160) > at > org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1.apply(JavaPairRDD.scala:1040) > at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) > at scala.collection.convert.Wrappers$IteratorWrapper.next(Wrappers.scala:31) > at com.google.common.collect.Lists.newArrayList(Lists.java:145) > at > org.apache.kylin.engine.spark.SparkFactDistinct$FlatOutputFucntion.call(SparkFactDistinct.java:313) > at > org.apache.kylin.engine.spark.SparkFactDistinct$FlatOutputFucntion.call(SparkFactDistinct.java:239) > at > org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:186) > at > org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:186) > at > org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) > at > org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:801) > at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) > at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96) > at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) > at org.apache.spark.scheduler.Task.run(Task.scala:109) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > > > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (KYLIN-3714) com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: Class is not registered: scala.reflect.ManifestFactory$$anon$2
[ https://issues.apache.org/jira/browse/KYLIN-3714?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16777940#comment-16777940 ] Alexander commented on KYLIN-3714: -- In my Env: CentOS 7 3 workers and 1 master. 4 cpu, 16GB RAM each and huge cube, with next settings: mapreduce.input.fileinputformat.split.maxsize 41943040 Hive concurrency lock kylin.engine.spark.rdd-partition-cut-mb=50 kylin.engine.spark.min-partition=1 kylin.engine.spark.max-partition=100 kylin.engine.mr.uhc-reducer-count=1 Still got this exception. I think that spark spilling process must be registered and working when dimensions weight is different. in my case, i have one dic 4.6 GB and other very small... with this settings i've files bigger than executor RAM avaliable. > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 > --- > > Key: KYLIN-3714 > URL: https://issues.apache.org/jira/browse/KYLIN-3714 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.1 > Environment: kylin2.5.1 > spark2.1.2 >Reporter: wangxianbin >Assignee: Chao Long >Priority: Major > Attachments: Class is not registered.png > > > Class is not registered in step "Extract Fact Table Distinct Columns", when > use spark as build engine, error as follow: > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 Note: To > register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); Serialization > trace: evidence$1 (org.apache.spark.util.collection.CompactBuffer) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:101) > at > com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518) > at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628) at > org.apache.spark.serializer.KryoSerializationStream.writeObject(KryoSerializer.scala:207) > at > org.apache.spark.serializer.SerializationStream.writeValue(Serializer.scala:135) > at > org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.org$apache$spark$util$collection$ExternalAppendOnlyMap$$spillMemoryIteratorToDisk(ExternalAppendOnlyMap.scala:227) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:186) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:56) > at org.apache.spark.util.collection.Spillable.maybeSpill(Spillable.scala:97) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.insertAll(ExternalAppendOnlyMap.scala:160) > at org.apache.spark.Aggregator.combineValuesByKey(Aggregator.scala:41) at > org.apache.spark.shuffle.BlockStoreShuffleReader.read(BlockStoreShuffleReader.scala:91) > at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:109) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at > org.apache.spark.scheduler.Task.run(Task.scala:99) at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) Caused by: > java.lang.IllegalArgumentException: Class is not registered: > scala.reflect.ManifestFactory$$anon$2 Note: To register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); at > com.esotericsoftware.kryo.Kryo.getRegistration(Kryo.java:488) at > com.esotericsoftware.kryo.util.DefaultClassResolver.writeClass(DefaultClassResolver.java:97) > at com.esotericsoftware.kryo.Kryo.writeClass(Kryo.java:517) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:76) > ... 27 more -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Reopened] (KYLIN-3714) com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: Class is not registered: scala.reflect.ManifestFactory$$anon$2
[ https://issues.apache.org/jira/browse/KYLIN-3714?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander reopened KYLIN-3714: -- Assignee: Alexander (was: Chao Long) > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 > --- > > Key: KYLIN-3714 > URL: https://issues.apache.org/jira/browse/KYLIN-3714 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.1 > Environment: kylin2.5.1 > spark2.1.2 >Reporter: wangxianbin >Assignee: Alexander >Priority: Major > Attachments: Class is not registered.png > > > Class is not registered in step "Extract Fact Table Distinct Columns", when > use spark as build engine, error as follow: > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 Note: To > register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); Serialization > trace: evidence$1 (org.apache.spark.util.collection.CompactBuffer) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:101) > at > com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518) > at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628) at > org.apache.spark.serializer.KryoSerializationStream.writeObject(KryoSerializer.scala:207) > at > org.apache.spark.serializer.SerializationStream.writeValue(Serializer.scala:135) > at > org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.org$apache$spark$util$collection$ExternalAppendOnlyMap$$spillMemoryIteratorToDisk(ExternalAppendOnlyMap.scala:227) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:186) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:56) > at org.apache.spark.util.collection.Spillable.maybeSpill(Spillable.scala:97) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.insertAll(ExternalAppendOnlyMap.scala:160) > at org.apache.spark.Aggregator.combineValuesByKey(Aggregator.scala:41) at > org.apache.spark.shuffle.BlockStoreShuffleReader.read(BlockStoreShuffleReader.scala:91) > at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:109) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at > org.apache.spark.scheduler.Task.run(Task.scala:99) at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) Caused by: > java.lang.IllegalArgumentException: Class is not registered: > scala.reflect.ManifestFactory$$anon$2 Note: To register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); at > com.esotericsoftware.kryo.Kryo.getRegistration(Kryo.java:488) at > com.esotericsoftware.kryo.util.DefaultClassResolver.writeClass(DefaultClassResolver.java:97) > at com.esotericsoftware.kryo.Kryo.writeClass(Kryo.java:517) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:76) > ... 27 more -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (KYLIN-3833) Potential OOM in Spark Extract Fact Table Distinct Columns step
[ https://issues.apache.org/jira/browse/KYLIN-3833?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16778100#comment-16778100 ] Alexander commented on KYLIN-3833: -- Tested on my Env. Ok. > Potential OOM in Spark Extract Fact Table Distinct Columns step > --- > > Key: KYLIN-3833 > URL: https://issues.apache.org/jira/browse/KYLIN-3833 > Project: Kylin > Issue Type: Bug > Components: Spark Engine >Affects Versions: v2.5.1 >Reporter: Alexander >Assignee: Alexander >Priority: Minor > Attachments: KYLIN-3833.master.001.patch > > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (KYLIN-3833) Potential OOM in Spark Extract Fact Table Distinct Columns step
[ https://issues.apache.org/jira/browse/KYLIN-3833?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3833: - Attachment: KYLIN-3833.master.001.patch > Potential OOM in Spark Extract Fact Table Distinct Columns step > --- > > Key: KYLIN-3833 > URL: https://issues.apache.org/jira/browse/KYLIN-3833 > Project: Kylin > Issue Type: Bug > Components: Spark Engine >Affects Versions: v2.5.1 >Reporter: Alexander >Assignee: Alexander >Priority: Minor > Attachments: KYLIN-3833.master.001.patch > > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (KYLIN-3833) Potential OOM in Spark Extract Fact Table Distinct Columns step
[ https://issues.apache.org/jira/browse/KYLIN-3833?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3833: - Environment: CentOS 7 3 workers and 1 master. 4 cpu, 16GB RAM each > Potential OOM in Spark Extract Fact Table Distinct Columns step > --- > > Key: KYLIN-3833 > URL: https://issues.apache.org/jira/browse/KYLIN-3833 > Project: Kylin > Issue Type: Bug > Components: Spark Engine >Affects Versions: v2.5.1 > Environment: CentOS 7 > 3 workers and 1 master. > 4 cpu, 16GB RAM each >Reporter: Alexander >Assignee: Alexander >Priority: Minor > Attachments: KYLIN-3833.master.001.patch > > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (KYLIN-3714) com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: Class is not registered: scala.reflect.ManifestFactory$$anon$2
[ https://issues.apache.org/jira/browse/KYLIN-3714?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16778105#comment-16778105 ] Alexander commented on KYLIN-3714: -- After patch was applied in my env, i test it. In my test case, i set properties kylin.engine.spark-conf.spark.driver.memory=2G kylin.engine.spark-conf.spark.executor.memory=512M Launch build a huge cube, with no mapreduce.input.fileinputformat.split at all. The most huge data goes to executor 20 (4.6G) and spark spilling process goes here: 2019-02-26 17:55:13 INFO ShuffleBlockFetcherIterator:54 - Getting 3898 non-empty blocks out of 3898 blocks 2019-02-26 17:55:13 INFO ShuffleBlockFetcherIterator:54 - Started 6 remote fetches in 39 ms 2019-02-26 17:55:15 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.5 MB to disk (1 time so far) 2019-02-26 17:55:40 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.5 MB to disk (2 times so far) 2019-02-26 17:56:02 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.6 MB to disk (3 times so far) 2019-02-26 17:56:25 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.5 MB to disk (4 times so far) 2019-02-26 17:56:54 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 99.2 MB to disk (5 times so far) 2019-02-26 17:57:18 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.5 MB to disk (6 times so far) 2019-02-26 17:57:42 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 98.4 MB to disk (7 times so far) 2019-02-26 17:58:09 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 99.2 MB to disk (8 times so far) 2019-02-26 17:58:34 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.5 MB to disk (9 times so far) 2019-02-26 17:58:54 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.5 MB to disk (10 times so far) 2019-02-26 17:59:17 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 99.2 MB to disk (11 times so far) 2019-02-26 17:59:46 INFO ExternalAppendOnlyMap:54 - Thread 34 spilling in-memory map of 97.5 MB to disk (12 times so far) 2019-02-26 18:00:13 INFO AbstractHadoopJob:511 - KylinConfig cached for : kylin_metadata@hdfs,path=hdfs://apachai1.apm.local:8020/kylin/kylin_metadata/kylin-5774d00a-bd56-ac28-e867-f9f5cb5d24f3/Test_Cube_2_clone/metadata 2019-02-26 18:00:13 INFO SparkFactDistinct:707 - Partition 19 handling column DEFAULT.SIDA_CASHIER_1.CASHIER_NAME, buildDictInReducer=true 2019-02-26 18:00:13 INFO SparkFactDistinct:716 - Received value: Николенко Наталья Николае Step finished succesfully. > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 > --- > > Key: KYLIN-3714 > URL: https://issues.apache.org/jira/browse/KYLIN-3714 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.1 > Environment: kylin2.5.1 > spark2.1.2 >Reporter: wangxianbin >Assignee: Alexander >Priority: Major > Attachments: Class is not registered.png > > > Class is not registered in step "Extract Fact Table Distinct Columns", when > use spark as build engine, error as follow: > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 Note: To > register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); Serialization > trace: evidence$1 (org.apache.spark.util.collection.CompactBuffer) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:101) > at > com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518) > at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628) at > org.apache.spark.serializer.KryoSerializationStream.writeObject(KryoSerializer.scala:207) > at > org.apache.spark.serializer.SerializationStream.writeValue(Serializer.scala:135) > at > org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.org$apache$spark$util$collection$ExternalAppendOnlyMap$$spillMemoryIteratorToDisk(ExternalAppendOnlyMap.scala:227) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:186) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:56) > at org.apache.spark.util.collection.Spillable.maybeSpill(Spillable.scala:97) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.insertAll(ExternalAppendOnl
[jira] [Updated] (KYLIN-3714) com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: Class is not registered: scala.reflect.ManifestFactory$$anon$2
[ https://issues.apache.org/jira/browse/KYLIN-3714?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3714: - Attachment: KYLIN-3714.master.001.patch > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 > --- > > Key: KYLIN-3714 > URL: https://issues.apache.org/jira/browse/KYLIN-3714 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.1 > Environment: kylin2.5.1 > spark2.1.2 >Reporter: wangxianbin >Assignee: Alexander >Priority: Major > Attachments: Class is not registered.png, KYLIN-3714.master.001.patch > > > Class is not registered in step "Extract Fact Table Distinct Columns", when > use spark as build engine, error as follow: > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 Note: To > register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); Serialization > trace: evidence$1 (org.apache.spark.util.collection.CompactBuffer) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:101) > at > com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518) > at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628) at > org.apache.spark.serializer.KryoSerializationStream.writeObject(KryoSerializer.scala:207) > at > org.apache.spark.serializer.SerializationStream.writeValue(Serializer.scala:135) > at > org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.org$apache$spark$util$collection$ExternalAppendOnlyMap$$spillMemoryIteratorToDisk(ExternalAppendOnlyMap.scala:227) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:186) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:56) > at org.apache.spark.util.collection.Spillable.maybeSpill(Spillable.scala:97) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.insertAll(ExternalAppendOnlyMap.scala:160) > at org.apache.spark.Aggregator.combineValuesByKey(Aggregator.scala:41) at > org.apache.spark.shuffle.BlockStoreShuffleReader.read(BlockStoreShuffleReader.scala:91) > at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:109) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at > org.apache.spark.scheduler.Task.run(Task.scala:99) at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) Caused by: > java.lang.IllegalArgumentException: Class is not registered: > scala.reflect.ManifestFactory$$anon$2 Note: To register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); at > com.esotericsoftware.kryo.Kryo.getRegistration(Kryo.java:488) at > com.esotericsoftware.kryo.util.DefaultClassResolver.writeClass(DefaultClassResolver.java:97) > at com.esotericsoftware.kryo.Kryo.writeClass(Kryo.java:517) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:76) > ... 27 more -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Created] (KYLIN-3833) Potential OOM in Spark Extract Fact Table Distinct Columns step
Alexander created KYLIN-3833: Summary: Potential OOM in Spark Extract Fact Table Distinct Columns step Key: KYLIN-3833 URL: https://issues.apache.org/jira/browse/KYLIN-3833 Project: Kylin Issue Type: Bug Components: Spark Engine Affects Versions: v2.5.1 Reporter: Alexander Assignee: Alexander -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (KYLIN-3833) Potential OOM in Spark Extract Fact Table Distinct Columns step
[ https://issues.apache.org/jira/browse/KYLIN-3833?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander resolved KYLIN-3833. -- Resolution: Fixed > Potential OOM in Spark Extract Fact Table Distinct Columns step > --- > > Key: KYLIN-3833 > URL: https://issues.apache.org/jira/browse/KYLIN-3833 > Project: Kylin > Issue Type: Bug > Components: Spark Engine >Affects Versions: v2.5.1 > Environment: CentOS 7 > 3 workers and 1 master. > 4 cpu, 16GB RAM each >Reporter: Alexander >Assignee: Alexander >Priority: Minor > Fix For: v2.6.1 > > Attachments: KYLIN-3833.master.001.patch > > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (KYLIN-3714) com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: Class is not registered: scala.reflect.ManifestFactory$$anon$2
[ https://issues.apache.org/jira/browse/KYLIN-3714?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander resolved KYLIN-3714. -- Resolution: Fixed > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 > --- > > Key: KYLIN-3714 > URL: https://issues.apache.org/jira/browse/KYLIN-3714 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.1 > Environment: kylin2.5.1 > spark2.1.2 >Reporter: wangxianbin >Assignee: Alexander >Priority: Major > Fix For: v2.6.1 > > Attachments: Class is not registered.png, KYLIN-3714.master.001.patch > > > Class is not registered in step "Extract Fact Table Distinct Columns", when > use spark as build engine, error as follow: > com.esotericsoftware.kryo.KryoException: java.lang.IllegalArgumentException: > Class is not registered: scala.reflect.ManifestFactory$$anon$2 Note: To > register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); Serialization > trace: evidence$1 (org.apache.spark.util.collection.CompactBuffer) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:101) > at > com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518) > at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628) at > org.apache.spark.serializer.KryoSerializationStream.writeObject(KryoSerializer.scala:207) > at > org.apache.spark.serializer.SerializationStream.writeValue(Serializer.scala:135) > at > org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.org$apache$spark$util$collection$ExternalAppendOnlyMap$$spillMemoryIteratorToDisk(ExternalAppendOnlyMap.scala:227) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:186) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.spill(ExternalAppendOnlyMap.scala:56) > at org.apache.spark.util.collection.Spillable.maybeSpill(Spillable.scala:97) > at > org.apache.spark.util.collection.ExternalAppendOnlyMap.insertAll(ExternalAppendOnlyMap.scala:160) > at org.apache.spark.Aggregator.combineValuesByKey(Aggregator.scala:41) at > org.apache.spark.shuffle.BlockStoreShuffleReader.read(BlockStoreShuffleReader.scala:91) > at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:109) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at > org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) at > org.apache.spark.rdd.RDD.iterator(RDD.scala:287) at > org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at > org.apache.spark.scheduler.Task.run(Task.scala:99) at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > at java.lang.Thread.run(Thread.java:745) Caused by: > java.lang.IllegalArgumentException: Class is not registered: > scala.reflect.ManifestFactory$$anon$2 Note: To register this class use: > kryo.register(scala.reflect.ManifestFactory$$anon$2.class); at > com.esotericsoftware.kryo.Kryo.getRegistration(Kryo.java:488) at > com.esotericsoftware.kryo.util.DefaultClassResolver.writeClass(DefaultClassResolver.java:97) > at com.esotericsoftware.kryo.Kryo.writeClass(Kryo.java:517) at > com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:76) > ... 27 more -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (KYLIN-3727) kylin2.5.2 不能构建空的cube,否则出现 HadoopShellException异常
[ https://issues.apache.org/jira/browse/KYLIN-3727?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16779680#comment-16779680 ] Alexander commented on KYLIN-3727: -- Set up system cube follow by instructions. - Got exception. (Fresh cube build will be empty). > kylin2.5.2 不能构建空的cube,否则出现 HadoopShellException异常 > -- > > Key: KYLIN-3727 > URL: https://issues.apache.org/jira/browse/KYLIN-3727 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.2 >Reporter: shiyu >Assignee: Lijun Cao >Priority: Major > Labels: Kylin > Attachments: image-2018-12-19-08-38-19-791.png > > > 注:kylin 已经开启,是否容忍数据源为空，默认值为 TRUE > kylin.job.allow-empty-segment=true > 出错位置:#12 Step Name: Load HFile to HBase Table > !image-2018-12-19-08-38-19-791.png! > 异常信息:HadoopShellException > org.apache.kylin.engine.mr.exception.HadoopShellException: result code:-1 > at > org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:73) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:70) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:113) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Assigned] (KYLIN-3727) kylin2.5.2 不能构建空的cube,否则出现 HadoopShellException异常
[ https://issues.apache.org/jira/browse/KYLIN-3727?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander reassigned KYLIN-3727: Assignee: Alexander (was: Lijun Cao) > kylin2.5.2 不能构建空的cube,否则出现 HadoopShellException异常 > -- > > Key: KYLIN-3727 > URL: https://issues.apache.org/jira/browse/KYLIN-3727 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.2 >Reporter: shiyu >Assignee: Alexander >Priority: Major > Labels: Kylin > Attachments: KYLIN-3727.master.001.patch, > image-2018-12-19-08-38-19-791.png > > > 注:kylin 已经开启,是否容忍数据源为空，默认值为 TRUE > kylin.job.allow-empty-segment=true > 出错位置:#12 Step Name: Load HFile to HBase Table > !image-2018-12-19-08-38-19-791.png! > 异常信息:HadoopShellException > org.apache.kylin.engine.mr.exception.HadoopShellException: result code:-1 > at > org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:73) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:70) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:113) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (KYLIN-3727) kylin2.5.2 不能构建空的cube,否则出现 HadoopShellException异常
[ https://issues.apache.org/jira/browse/KYLIN-3727?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3727: - Attachment: KYLIN-3727.master.001.patch > kylin2.5.2 不能构建空的cube,否则出现 HadoopShellException异常 > -- > > Key: KYLIN-3727 > URL: https://issues.apache.org/jira/browse/KYLIN-3727 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.2 >Reporter: shiyu >Assignee: Lijun Cao >Priority: Major > Labels: Kylin > Attachments: KYLIN-3727.master.001.patch, > image-2018-12-19-08-38-19-791.png > > > 注:kylin 已经开启,是否容忍数据源为空，默认值为 TRUE > kylin.job.allow-empty-segment=true > 出错位置:#12 Step Name: Load HFile to HBase Table > !image-2018-12-19-08-38-19-791.png! > 异常信息:HadoopShellException > org.apache.kylin.engine.mr.exception.HadoopShellException: result code:-1 > at > org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:73) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:70) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:113) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (KYLIN-3727) Can not build empty segment, HadoopShellException
[ https://issues.apache.org/jira/browse/KYLIN-3727?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander resolved KYLIN-3727. -- Resolution: Fixed > Can not build empty segment, HadoopShellException > - > > Key: KYLIN-3727 > URL: https://issues.apache.org/jira/browse/KYLIN-3727 > Project: Kylin > Issue Type: Bug > Components: Job Engine >Affects Versions: v2.5.2 >Reporter: shiyu >Assignee: Alexander >Priority: Major > Labels: Kylin > Fix For: v2.6.1 > > Attachments: KYLIN-3727.master.001.patch, > image-2018-12-19-08-38-19-791.png > > > 注:kylin 已经开启,是否容忍数据源为空，默认值为 TRUE > kylin.job.allow-empty-segment=true > 出错位置:#12 Step Name: Load HFile to HBase Table > !image-2018-12-19-08-38-19-791.png! > 异常信息:HadoopShellException > org.apache.kylin.engine.mr.exception.HadoopShellException: result code:-1 > at > org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:73) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:70) > at > org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:164) > at > org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:113) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Created] (KYLIN-3965) When using DriverManager - No suitable driver found for jdbc:kylin://
Alexander created KYLIN-3965: Summary: When using DriverManager - No suitable driver found for jdbc:kylin:// Key: KYLIN-3965 URL: https://issues.apache.org/jira/browse/KYLIN-3965 Project: Kylin Issue Type: Bug Components: Driver - JDBC Affects Versions: v2.6.1 Reporter: Alexander Assignee: Alexander Caused by: java.sql.SQLException: No suitable driver found for jdbc:kylin:// This is because META-INF/services/java.sql.Driver got incorrect name org.apache.calcite.avatica.remote.Driver -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Updated] (KYLIN-3965) When using DriverManager - No suitable driver found for jdbc:kylin://
[ https://issues.apache.org/jira/browse/KYLIN-3965?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3965: - Attachment: KYLIN-3965.master.001.patch > When using DriverManager - No suitable driver found for jdbc:kylin:// > - > > Key: KYLIN-3965 > URL: https://issues.apache.org/jira/browse/KYLIN-3965 > Project: Kylin > Issue Type: Bug > Components: Driver - JDBC >Affects Versions: v2.6.1 >Reporter: Alexander >Assignee: Alexander >Priority: Minor > Attachments: KYLIN-3965.master.001.patch > > > Caused by: java.sql.SQLException: No suitable driver found for jdbc:kylin:// > > This is because META-INF/services/java.sql.Driver got incorrect name > org.apache.calcite.avatica.remote.Driver -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (KYLIN-3965) When using DriverManager - No suitable driver found for jdbc:kylin://
[ https://issues.apache.org/jira/browse/KYLIN-3965?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander resolved KYLIN-3965. -- Resolution: Fixed PR merged. > When using DriverManager - No suitable driver found for jdbc:kylin:// > - > > Key: KYLIN-3965 > URL: https://issues.apache.org/jira/browse/KYLIN-3965 > Project: Kylin > Issue Type: Bug > Components: Driver - JDBC >Affects Versions: v2.6.1 >Reporter: Alexander >Assignee: Alexander >Priority: Minor > Fix For: v2.6.2 > > Attachments: KYLIN-3965.master.001.patch > > > Caused by: java.sql.SQLException: No suitable driver found for jdbc:kylin:// > > This is because META-INF/services/java.sql.Driver got incorrect name > org.apache.calcite.avatica.remote.Driver -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Created] (KYLIN-4038) Lookup tables is empty whn build on ORC tables.
Alexander created KYLIN-4038:
Summary: Lookup tables is empty whn build on ORC tables.
Key: KYLIN-4038
URL: https://issues.apache.org/jira/browse/KYLIN-4038
Project: Kylin
Issue Type: Bug
Affects Versions: v2.6.2
Environment: HDP 3 cluster.
Reporter: Alexander
Create tables
{code:java}
// Create tables
create table kylin_sales_sida (trans_id bigint,part_dt date,lstg_format_name
string,leaf_categ_id bigint,lstg_site_id int,slr_segment_cd smallint,price
decimal(19,4),item_count bigint,seller_id bigint,buyer_id bigint,ops_user_id
string,ops_region string) stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_sales_sida select * from kylin_sales;
create table kylin_country_sida (country string,latitude double,longitude
double,name string) stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_country_sida select * from kylin_country;
create table kylin_category_groupings_sida (leaf_categ_id
bigint,leaf_categ_name string,site_id int,categ_busn_mgr string,categ_busn_unit
string,regn_categ string,user_defined_field1 string,user_defined_field3
string,kylin_groupings_cre_date string,kylin_groupings_upd_date
string,kylin_groupings_cre_user string,kylin_groupings_upd_user
string,meta_categ_id decimal(10,0),meta_categ_name string,categ_lvl2_id
decimal(10,0),categ_lvl3_id decimal(10,0),categ_lvl4_id
decimal(10,0),categ_lvl5_id decimal(10,0),categ_lvl6_id
decimal(10,0),categ_lvl7_id decimal(10,0),categ_lvl2_name
string,categ_lvl3_name string,categ_lvl4_name string,categ_lvl5_name
string,categ_lvl6_name string,categ_lvl7_name string,categ_flags
decimal(10,0),adult_categ_yn string,domain_id decimal(10,0),user_defined_field5
string,vcs_id decimal(10,0),gcs_id decimal(10,0),move_to
decimal(10,0),sap_category_id decimal(10,0),src_id tinyint,bsns_vrtcl_name
string) stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_category_groupings_sida select * from
kylin_category_groupings;
create table kylin_cal_dt_sida (cal_dt date,year_beg_dt date,qtr_beg_dt
date,month_beg_dt date,week_beg_dt date,age_for_year_id smallint,age_for_qtr_id
smallint,age_for_month_id smallint,age_for_week_id smallint,age_for_dt_id
smallint,age_for_rtl_year_id smallint,age_for_rtl_qtr_id
smallint,age_for_rtl_month_id smallint,age_for_rtl_week_id
smallint,age_for_cs_week_id smallint,day_of_cal_id int,day_of_year_id
smallint,day_of_qtr_id smallint,day_of_month_id smallint,day_of_week_id
int,week_of_year_id tinyint,week_of_cal_id int,month_of_qtr_id
tinyint,month_of_year_id tinyint,month_of_cal_id smallint,qtr_of_year_id
tinyint,qtr_of_cal_id smallint,year_of_cal_id smallint,year_end_dt
string,qtr_end_dt string,month_end_dt string,week_end_dt string,cal_dt_name
string,cal_dt_desc string,cal_dt_short_name string,ytd_yn_id tinyint,qtd_yn_id
tinyint,mtd_yn_id tinyint,wtd_yn_id tinyint,season_beg_dt
string,day_in_year_count smallint,day_in_qtr_count tinyint,day_in_month_count
tinyint,day_in_week_count tinyint,rtl_year_beg_dt string,rtl_qtr_beg_dt
string,rtl_month_beg_dt string,rtl_week_beg_dt string,cs_week_beg_dt
string,cal_date string,day_of_week string,month_id string,prd_desc
string,prd_flag string,prd_id string,prd_ind string,qtr_desc string,qtr_id
string,qtr_ind string,retail_week string,retail_year string,retail_start_date
string,retail_wk_end_date string,week_ind string,week_num_desc
string,week_beg_date string,week_end_date string,week_in_year_id string,week_id
string,week_beg_end_desc_mdy string,week_beg_end_desc_md string,year_id
string,year_ind string,cal_dt_mns_1year_dt string,cal_dt_mns_2year_dt
string,cal_dt_mns_1qtr_dt string,cal_dt_mns_2qtr_dt string,cal_dt_mns_1month_dt
string,cal_dt_mns_2month_dt string,cal_dt_mns_1week_dt
string,cal_dt_mns_2week_dt string,curr_cal_dt_mns_1year_yn_id
tinyint,curr_cal_dt_mns_2year_yn_id tinyint,curr_cal_dt_mns_1qtr_yn_id
tinyint,curr_cal_dt_mns_2qtr_yn_id tinyint,curr_cal_dt_mns_1month_yn_id
tinyint,curr_cal_dt_mns_2month_yn_id tinyint,curr_cal_dt_mns_1week_yn_ind
tinyint,curr_cal_dt_mns_2week_yn_ind tinyint,rtl_month_of_rtl_year_id
string,rtl_qtr_of_rtl_year_id tinyint,rtl_week_of_rtl_year_id
tinyint,season_of_year_id tinyint,ytm_yn_id tinyint,ytq_yn_id tinyint,ytw_yn_id
tinyint,kylin_cal_dt_cre_date string,kylin_cal_dt_cre_user
string,kylin_cal_dt_upd_date string,kylin_cal_dt_upd_user string) stored as orc
tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_cal_dt_sida select * from kylin_cal_dt;
create table kylin_account_sida (account_id bigint,account_buyer_level
int,account_seller_level int,account_country string,account_contact string)
stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_account_sida select * from kylin_account;{code}
Create new project and data model, a manual copy of test cube learn
[jira] [Updated] (KYLIN-4038) Lookup tables is empty when build on ORC tables.
[
https://issues.apache.org/jira/browse/KYLIN-4038?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4038:
-
Summary: Lookup tables is empty when build on ORC tables. (was: Lookup
tables is empty whn build on ORC tables.)
> Lookup tables is empty when build on ORC tables.
>
>
> Key: KYLIN-4038
> URL: https://issues.apache.org/jira/browse/KYLIN-4038
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v2.6.2
> Environment: HDP 3 cluster.
>Reporter: Alexander
>Priority: Blocker
>
> Create tables
> {code:java}
> // Create tables
> create table kylin_sales_sida (trans_id bigint,part_dt date,lstg_format_name
> string,leaf_categ_id bigint,lstg_site_id int,slr_segment_cd smallint,price
> decimal(19,4),item_count bigint,seller_id bigint,buyer_id bigint,ops_user_id
> string,ops_region string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_sales_sida select * from kylin_sales;
> create table kylin_country_sida (country string,latitude double,longitude
> double,name string) stored as orc tblproperties ("orc.compress"="SNAPPY");
> insert into table kylin_country_sida select * from kylin_country;
> create table kylin_category_groupings_sida (leaf_categ_id
> bigint,leaf_categ_name string,site_id int,categ_busn_mgr
> string,categ_busn_unit string,regn_categ string,user_defined_field1
> string,user_defined_field3 string,kylin_groupings_cre_date
> string,kylin_groupings_upd_date string,kylin_groupings_cre_user
> string,kylin_groupings_upd_user string,meta_categ_id
> decimal(10,0),meta_categ_name string,categ_lvl2_id
> decimal(10,0),categ_lvl3_id decimal(10,0),categ_lvl4_id
> decimal(10,0),categ_lvl5_id decimal(10,0),categ_lvl6_id
> decimal(10,0),categ_lvl7_id decimal(10,0),categ_lvl2_name
> string,categ_lvl3_name string,categ_lvl4_name string,categ_lvl5_name
> string,categ_lvl6_name string,categ_lvl7_name string,categ_flags
> decimal(10,0),adult_categ_yn string,domain_id
> decimal(10,0),user_defined_field5 string,vcs_id decimal(10,0),gcs_id
> decimal(10,0),move_to decimal(10,0),sap_category_id decimal(10,0),src_id
> tinyint,bsns_vrtcl_name string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_category_groupings_sida select * from
> kylin_category_groupings;
> create table kylin_cal_dt_sida (cal_dt date,year_beg_dt date,qtr_beg_dt
> date,month_beg_dt date,week_beg_dt date,age_for_year_id
> smallint,age_for_qtr_id smallint,age_for_month_id smallint,age_for_week_id
> smallint,age_for_dt_id smallint,age_for_rtl_year_id
> smallint,age_for_rtl_qtr_id smallint,age_for_rtl_month_id
> smallint,age_for_rtl_week_id smallint,age_for_cs_week_id
> smallint,day_of_cal_id int,day_of_year_id smallint,day_of_qtr_id
> smallint,day_of_month_id smallint,day_of_week_id int,week_of_year_id
> tinyint,week_of_cal_id int,month_of_qtr_id tinyint,month_of_year_id
> tinyint,month_of_cal_id smallint,qtr_of_year_id tinyint,qtr_of_cal_id
> smallint,year_of_cal_id smallint,year_end_dt string,qtr_end_dt
> string,month_end_dt string,week_end_dt string,cal_dt_name string,cal_dt_desc
> string,cal_dt_short_name string,ytd_yn_id tinyint,qtd_yn_id tinyint,mtd_yn_id
> tinyint,wtd_yn_id tinyint,season_beg_dt string,day_in_year_count
> smallint,day_in_qtr_count tinyint,day_in_month_count
> tinyint,day_in_week_count tinyint,rtl_year_beg_dt string,rtl_qtr_beg_dt
> string,rtl_month_beg_dt string,rtl_week_beg_dt string,cs_week_beg_dt
> string,cal_date string,day_of_week string,month_id string,prd_desc
> string,prd_flag string,prd_id string,prd_ind string,qtr_desc string,qtr_id
> string,qtr_ind string,retail_week string,retail_year string,retail_start_date
> string,retail_wk_end_date string,week_ind string,week_num_desc
> string,week_beg_date string,week_end_date string,week_in_year_id
> string,week_id string,week_beg_end_desc_mdy string,week_beg_end_desc_md
> string,year_id string,year_ind string,cal_dt_mns_1year_dt
> string,cal_dt_mns_2year_dt string,cal_dt_mns_1qtr_dt
> string,cal_dt_mns_2qtr_dt string,cal_dt_mns_1month_dt
> string,cal_dt_mns_2month_dt string,cal_dt_mns_1week_dt
> string,cal_dt_mns_2week_dt string,curr_cal_dt_mns_1year_yn_id
> tinyint,curr_cal_dt_mns_2year_yn_id tinyint,curr_cal_dt_mns_1qtr_yn_id
> tinyint,curr_cal_dt_mns_2qtr_yn_id tinyint,curr_cal_dt_mns_1month_yn_id
> tinyint,curr_cal_dt_mns_2month_yn_id tinyint,curr_cal_dt_mns_1week_yn_ind
> tinyint,curr_cal_dt_mns_2week_yn_ind tinyint,rtl_month_of_rtl_year_id
> string,rtl_qtr_of_rtl_year_id tinyint,rtl_week_of_rtl_year_id
> tinyint,season_of_year_id tinyint,ytm_yn_id tinyint,ytq_yn_id
> tinyint,ytw_yn_id tinyint,kylin_cal_dt_cre_date string,kylin_cal_dt_cre_user
> string,kylin_cal_dt_upd_date string,kylin_cal_dt_upd_user string) stored as
> orc
[jira] [Updated] (KYLIN-4038) Lookup tables is empty when build on ORC tables.
[
https://issues.apache.org/jira/browse/KYLIN-4038?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4038:
-
Description:
Create tables, simple copy of learn_kylin project.
{code:java}
// Create tables
create table kylin_sales_sida (trans_id bigint,part_dt date,lstg_format_name
string,leaf_categ_id bigint,lstg_site_id int,slr_segment_cd smallint,price
decimal(19,4),item_count bigint,seller_id bigint,buyer_id bigint,ops_user_id
string,ops_region string) stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_sales_sida select * from kylin_sales;
create table kylin_country_sida (country string,latitude double,longitude
double,name string) stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_country_sida select * from kylin_country;
create table kylin_category_groupings_sida (leaf_categ_id
bigint,leaf_categ_name string,site_id int,categ_busn_mgr string,categ_busn_unit
string,regn_categ string,user_defined_field1 string,user_defined_field3
string,kylin_groupings_cre_date string,kylin_groupings_upd_date
string,kylin_groupings_cre_user string,kylin_groupings_upd_user
string,meta_categ_id decimal(10,0),meta_categ_name string,categ_lvl2_id
decimal(10,0),categ_lvl3_id decimal(10,0),categ_lvl4_id
decimal(10,0),categ_lvl5_id decimal(10,0),categ_lvl6_id
decimal(10,0),categ_lvl7_id decimal(10,0),categ_lvl2_name
string,categ_lvl3_name string,categ_lvl4_name string,categ_lvl5_name
string,categ_lvl6_name string,categ_lvl7_name string,categ_flags
decimal(10,0),adult_categ_yn string,domain_id decimal(10,0),user_defined_field5
string,vcs_id decimal(10,0),gcs_id decimal(10,0),move_to
decimal(10,0),sap_category_id decimal(10,0),src_id tinyint,bsns_vrtcl_name
string) stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_category_groupings_sida select * from
kylin_category_groupings;
create table kylin_cal_dt_sida (cal_dt date,year_beg_dt date,qtr_beg_dt
date,month_beg_dt date,week_beg_dt date,age_for_year_id smallint,age_for_qtr_id
smallint,age_for_month_id smallint,age_for_week_id smallint,age_for_dt_id
smallint,age_for_rtl_year_id smallint,age_for_rtl_qtr_id
smallint,age_for_rtl_month_id smallint,age_for_rtl_week_id
smallint,age_for_cs_week_id smallint,day_of_cal_id int,day_of_year_id
smallint,day_of_qtr_id smallint,day_of_month_id smallint,day_of_week_id
int,week_of_year_id tinyint,week_of_cal_id int,month_of_qtr_id
tinyint,month_of_year_id tinyint,month_of_cal_id smallint,qtr_of_year_id
tinyint,qtr_of_cal_id smallint,year_of_cal_id smallint,year_end_dt
string,qtr_end_dt string,month_end_dt string,week_end_dt string,cal_dt_name
string,cal_dt_desc string,cal_dt_short_name string,ytd_yn_id tinyint,qtd_yn_id
tinyint,mtd_yn_id tinyint,wtd_yn_id tinyint,season_beg_dt
string,day_in_year_count smallint,day_in_qtr_count tinyint,day_in_month_count
tinyint,day_in_week_count tinyint,rtl_year_beg_dt string,rtl_qtr_beg_dt
string,rtl_month_beg_dt string,rtl_week_beg_dt string,cs_week_beg_dt
string,cal_date string,day_of_week string,month_id string,prd_desc
string,prd_flag string,prd_id string,prd_ind string,qtr_desc string,qtr_id
string,qtr_ind string,retail_week string,retail_year string,retail_start_date
string,retail_wk_end_date string,week_ind string,week_num_desc
string,week_beg_date string,week_end_date string,week_in_year_id string,week_id
string,week_beg_end_desc_mdy string,week_beg_end_desc_md string,year_id
string,year_ind string,cal_dt_mns_1year_dt string,cal_dt_mns_2year_dt
string,cal_dt_mns_1qtr_dt string,cal_dt_mns_2qtr_dt string,cal_dt_mns_1month_dt
string,cal_dt_mns_2month_dt string,cal_dt_mns_1week_dt
string,cal_dt_mns_2week_dt string,curr_cal_dt_mns_1year_yn_id
tinyint,curr_cal_dt_mns_2year_yn_id tinyint,curr_cal_dt_mns_1qtr_yn_id
tinyint,curr_cal_dt_mns_2qtr_yn_id tinyint,curr_cal_dt_mns_1month_yn_id
tinyint,curr_cal_dt_mns_2month_yn_id tinyint,curr_cal_dt_mns_1week_yn_ind
tinyint,curr_cal_dt_mns_2week_yn_ind tinyint,rtl_month_of_rtl_year_id
string,rtl_qtr_of_rtl_year_id tinyint,rtl_week_of_rtl_year_id
tinyint,season_of_year_id tinyint,ytm_yn_id tinyint,ytq_yn_id tinyint,ytw_yn_id
tinyint,kylin_cal_dt_cre_date string,kylin_cal_dt_cre_user
string,kylin_cal_dt_upd_date string,kylin_cal_dt_upd_user string) stored as orc
tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_cal_dt_sida select * from kylin_cal_dt;
create table kylin_account_sida (account_id bigint,account_buyer_level
int,account_seller_level int,account_country string,account_contact string)
stored as orc tblproperties ("orc.compress"="SNAPPY");
insert into table kylin_account_sida select * from kylin_account;{code}
Create new project and data model, a manual copy of test cube learn_kylin. Add
Lookup tables to HBase.
Build cube, and notice what the lookup tables builds empty:
{code:java}
// Result of Convert Lookup Table to HF
[jira] [Commented] (KYLIN-4038) Lookup tables is empty when build on ORC tables.
[
https://issues.apache.org/jira/browse/KYLIN-4038?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16860603#comment-16860603
]
Alexander commented on KYLIN-4038:
--
Hello,
In my case i use MANAGED ORC tables.
As i see, managed orc tables not working in Spark too...
Is it any kind of limitation of Hive tables usage?
> Lookup tables is empty when build on ORC tables.
>
>
> Key: KYLIN-4038
> URL: https://issues.apache.org/jira/browse/KYLIN-4038
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v2.6.2
> Environment: HDP 3 cluster.
>Reporter: Alexander
>Assignee: XiaoXiang Yu
>Priority: Blocker
>
> Create tables, simple copy of learn_kylin project.
> {code:java}
> // Create tables
> create table kylin_sales_sida (trans_id bigint,part_dt date,lstg_format_name
> string,leaf_categ_id bigint,lstg_site_id int,slr_segment_cd smallint,price
> decimal(19,4),item_count bigint,seller_id bigint,buyer_id bigint,ops_user_id
> string,ops_region string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_sales_sida select * from kylin_sales;
> create table kylin_country_sida (country string,latitude double,longitude
> double,name string) stored as orc tblproperties ("orc.compress"="SNAPPY");
> insert into table kylin_country_sida select * from kylin_country;
> create table kylin_category_groupings_sida (leaf_categ_id
> bigint,leaf_categ_name string,site_id int,categ_busn_mgr
> string,categ_busn_unit string,regn_categ string,user_defined_field1
> string,user_defined_field3 string,kylin_groupings_cre_date
> string,kylin_groupings_upd_date string,kylin_groupings_cre_user
> string,kylin_groupings_upd_user string,meta_categ_id
> decimal(10,0),meta_categ_name string,categ_lvl2_id
> decimal(10,0),categ_lvl3_id decimal(10,0),categ_lvl4_id
> decimal(10,0),categ_lvl5_id decimal(10,0),categ_lvl6_id
> decimal(10,0),categ_lvl7_id decimal(10,0),categ_lvl2_name
> string,categ_lvl3_name string,categ_lvl4_name string,categ_lvl5_name
> string,categ_lvl6_name string,categ_lvl7_name string,categ_flags
> decimal(10,0),adult_categ_yn string,domain_id
> decimal(10,0),user_defined_field5 string,vcs_id decimal(10,0),gcs_id
> decimal(10,0),move_to decimal(10,0),sap_category_id decimal(10,0),src_id
> tinyint,bsns_vrtcl_name string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_category_groupings_sida select * from
> kylin_category_groupings;
> create table kylin_cal_dt_sida (cal_dt date,year_beg_dt date,qtr_beg_dt
> date,month_beg_dt date,week_beg_dt date,age_for_year_id
> smallint,age_for_qtr_id smallint,age_for_month_id smallint,age_for_week_id
> smallint,age_for_dt_id smallint,age_for_rtl_year_id
> smallint,age_for_rtl_qtr_id smallint,age_for_rtl_month_id
> smallint,age_for_rtl_week_id smallint,age_for_cs_week_id
> smallint,day_of_cal_id int,day_of_year_id smallint,day_of_qtr_id
> smallint,day_of_month_id smallint,day_of_week_id int,week_of_year_id
> tinyint,week_of_cal_id int,month_of_qtr_id tinyint,month_of_year_id
> tinyint,month_of_cal_id smallint,qtr_of_year_id tinyint,qtr_of_cal_id
> smallint,year_of_cal_id smallint,year_end_dt string,qtr_end_dt
> string,month_end_dt string,week_end_dt string,cal_dt_name string,cal_dt_desc
> string,cal_dt_short_name string,ytd_yn_id tinyint,qtd_yn_id tinyint,mtd_yn_id
> tinyint,wtd_yn_id tinyint,season_beg_dt string,day_in_year_count
> smallint,day_in_qtr_count tinyint,day_in_month_count
> tinyint,day_in_week_count tinyint,rtl_year_beg_dt string,rtl_qtr_beg_dt
> string,rtl_month_beg_dt string,rtl_week_beg_dt string,cs_week_beg_dt
> string,cal_date string,day_of_week string,month_id string,prd_desc
> string,prd_flag string,prd_id string,prd_ind string,qtr_desc string,qtr_id
> string,qtr_ind string,retail_week string,retail_year string,retail_start_date
> string,retail_wk_end_date string,week_ind string,week_num_desc
> string,week_beg_date string,week_end_date string,week_in_year_id
> string,week_id string,week_beg_end_desc_mdy string,week_beg_end_desc_md
> string,year_id string,year_ind string,cal_dt_mns_1year_dt
> string,cal_dt_mns_2year_dt string,cal_dt_mns_1qtr_dt
> string,cal_dt_mns_2qtr_dt string,cal_dt_mns_1month_dt
> string,cal_dt_mns_2month_dt string,cal_dt_mns_1week_dt
> string,cal_dt_mns_2week_dt string,curr_cal_dt_mns_1year_yn_id
> tinyint,curr_cal_dt_mns_2year_yn_id tinyint,curr_cal_dt_mns_1qtr_yn_id
> tinyint,curr_cal_dt_mns_2qtr_yn_id tinyint,curr_cal_dt_mns_1month_yn_id
> tinyint,curr_cal_dt_mns_2month_yn_id tinyint,curr_cal_dt_mns_1week_yn_ind
> tinyint,curr_cal_dt_mns_2week_yn_ind tinyint,rtl_month_of_rtl_year_id
> string,rtl_qtr_of_rtl_year_id tinyint,rtl_week_of_rtl_year_id
> tinyint,season_of_year_id tinyint,ytm_yn_id tinyint,ytq_
[jira] [Commented] (KYLIN-4038) Lookup tables is empty when build on ORC tables.
[
https://issues.apache.org/jira/browse/KYLIN-4038?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16860676#comment-16860676
]
Alexander commented on KYLIN-4038:
--
Hello,
Yes, it's because ORC with Snappy have great compression...
learn_kylin tables is a MANAGED Text tables with insert_only mode and they are
have much more size.
if you perform select * from *tables*(orc), from beeline, you can see that all
rows persist in table.
But selection from orc tables works only in Beeline and hive clients. Select
from spark-shell shows empty table too (like a MR Job in Kylin):
{code:java}
// Spark
scala> spark.table("DEFAULT.KYLIN_ACCOUNT_SIDA").show
+--+---++---+---+
|account_id|account_buyer_level|account_seller_level|account_country|account_contact|
+--+---++---+---+
+--+---++---+---+
scala> spark.table("DEFAULT.KYLIN_ACCOUNT_SIDA").count
res21: Long = 0
{code}
Trying to select it thru JDBC as a workaround
(https://github.com/Gowthamsb12/Spark/blob/master/Spark_ACID), and no result
too:
{code:java}
// Spark JDBC call
import org.apache.spark.sql.jdbc.JdbcDialect
import org.apache.spark.sql.jdbc.JdbcDialects
object HiveDialect extends JdbcDialect {
override def canHandle(url : String): Boolean = url.startsWith("jdbc:hive2")
override def quoteIdentifier(colName: String): String = {
colName.split('.').map(part => s"`$part`").mkString(".")
}
}
JdbcDialects.registerDialect(HiveDialect)
scala> val jdbcDF = spark.read.format("jdbc").option("url",
"jdbc:hive2://localhost:1/").option("dbtable",
"DEFAULT.KYLIN_ACCOUNT_SIDA").option("user", "***").option("password",
"***").load()
jdbcDF: org.apache.spark.sql.DataFrame = [kylin_account_sida.account_id:
bigint, kylin_account_sida.account_buyer_level: int ... 3 more fields]
scala> jdbcDF.count
res28: Long = 0
scala> jdbcDF.show
+-+--+---+--+--+
|kylin_account_sida.account_id|kylin_account_sida.account_buyer_level|kylin_account_sida.account_seller_level|kylin_account_sida.account_country|kylin_account_sida.account_contact|
+-+--+---+--+--+
+-+--+---+--+--+
{code}
> Lookup tables is empty when build on ORC tables.
>
>
> Key: KYLIN-4038
> URL: https://issues.apache.org/jira/browse/KYLIN-4038
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v2.6.2
> Environment: HDP 3 cluster.
>Reporter: Alexander
>Assignee: XiaoXiang Yu
>Priority: Blocker
> Attachments: image-2019-06-11-16-12-09-472.png
>
>
> Create tables, simple copy of learn_kylin project.
> {code:java}
> // Create tables
> create table kylin_sales_sida (trans_id bigint,part_dt date,lstg_format_name
> string,leaf_categ_id bigint,lstg_site_id int,slr_segment_cd smallint,price
> decimal(19,4),item_count bigint,seller_id bigint,buyer_id bigint,ops_user_id
> string,ops_region string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_sales_sida select * from kylin_sales;
> create table kylin_country_sida (country string,latitude double,longitude
> double,name string) stored as orc tblproperties ("orc.compress"="SNAPPY");
> insert into table kylin_country_sida select * from kylin_country;
> create table kylin_category_groupings_sida (leaf_categ_id
> bigint,leaf_categ_name string,site_id int,categ_busn_mgr
> string,categ_busn_unit string,regn_categ string,user_defined_field1
> string,user_defined_field3 string,kylin_groupings_cre_date
> string,kylin_groupings_upd_date string,kylin_groupings_cre_user
> string,kylin_groupings_upd_user string,meta_categ_id
> decimal(10,0),meta_categ_name string,categ_lvl2_id
> decimal(10,0),categ_lvl3_id decimal(10,0),categ_lvl4_id
> decimal(10,0),categ_lvl5_id decimal(10,0),categ_lvl6_id
> decimal(10,0),categ_lvl7_id decimal(10,0),categ_lvl2_name
> string,categ_lvl3_name string,categ_lvl4_name string,categ_lvl5_name
> string,categ_lvl6_name string,categ_lvl7_name string,categ_flags
> decimal(10,0),adult_categ_yn string,domain_id
> decimal(10,0),user_defined_field5 string,vcs_id decimal(10,0),gcs_id
> decimal(10,0),move_to decimal(10,0),sap_category_id decimal(10,0),src_id
> tinyint,bsns_v
[jira] [Commented] (KYLIN-4038) Lookup tables is empty when build on ORC tables.
[
https://issues.apache.org/jira/browse/KYLIN-4038?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16860690#comment-16860690
]
Alexander commented on KYLIN-4038:
--
XiaoXiang Yu, many thanks for you research!
This is it.
I'm stuck at this point, and need any suggestions\help on that...
> Lookup tables is empty when build on ORC tables.
>
>
> Key: KYLIN-4038
> URL: https://issues.apache.org/jira/browse/KYLIN-4038
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v2.6.2
> Environment: HDP 3 cluster.
>Reporter: Alexander
>Assignee: XiaoXiang Yu
>Priority: Blocker
> Attachments: image-2019-06-11-16-12-09-472.png
>
>
> Create tables, simple copy of learn_kylin project.
> {code:java}
> // Create tables
> create table kylin_sales_sida (trans_id bigint,part_dt date,lstg_format_name
> string,leaf_categ_id bigint,lstg_site_id int,slr_segment_cd smallint,price
> decimal(19,4),item_count bigint,seller_id bigint,buyer_id bigint,ops_user_id
> string,ops_region string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_sales_sida select * from kylin_sales;
> create table kylin_country_sida (country string,latitude double,longitude
> double,name string) stored as orc tblproperties ("orc.compress"="SNAPPY");
> insert into table kylin_country_sida select * from kylin_country;
> create table kylin_category_groupings_sida (leaf_categ_id
> bigint,leaf_categ_name string,site_id int,categ_busn_mgr
> string,categ_busn_unit string,regn_categ string,user_defined_field1
> string,user_defined_field3 string,kylin_groupings_cre_date
> string,kylin_groupings_upd_date string,kylin_groupings_cre_user
> string,kylin_groupings_upd_user string,meta_categ_id
> decimal(10,0),meta_categ_name string,categ_lvl2_id
> decimal(10,0),categ_lvl3_id decimal(10,0),categ_lvl4_id
> decimal(10,0),categ_lvl5_id decimal(10,0),categ_lvl6_id
> decimal(10,0),categ_lvl7_id decimal(10,0),categ_lvl2_name
> string,categ_lvl3_name string,categ_lvl4_name string,categ_lvl5_name
> string,categ_lvl6_name string,categ_lvl7_name string,categ_flags
> decimal(10,0),adult_categ_yn string,domain_id
> decimal(10,0),user_defined_field5 string,vcs_id decimal(10,0),gcs_id
> decimal(10,0),move_to decimal(10,0),sap_category_id decimal(10,0),src_id
> tinyint,bsns_vrtcl_name string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_category_groupings_sida select * from
> kylin_category_groupings;
> create table kylin_cal_dt_sida (cal_dt date,year_beg_dt date,qtr_beg_dt
> date,month_beg_dt date,week_beg_dt date,age_for_year_id
> smallint,age_for_qtr_id smallint,age_for_month_id smallint,age_for_week_id
> smallint,age_for_dt_id smallint,age_for_rtl_year_id
> smallint,age_for_rtl_qtr_id smallint,age_for_rtl_month_id
> smallint,age_for_rtl_week_id smallint,age_for_cs_week_id
> smallint,day_of_cal_id int,day_of_year_id smallint,day_of_qtr_id
> smallint,day_of_month_id smallint,day_of_week_id int,week_of_year_id
> tinyint,week_of_cal_id int,month_of_qtr_id tinyint,month_of_year_id
> tinyint,month_of_cal_id smallint,qtr_of_year_id tinyint,qtr_of_cal_id
> smallint,year_of_cal_id smallint,year_end_dt string,qtr_end_dt
> string,month_end_dt string,week_end_dt string,cal_dt_name string,cal_dt_desc
> string,cal_dt_short_name string,ytd_yn_id tinyint,qtd_yn_id tinyint,mtd_yn_id
> tinyint,wtd_yn_id tinyint,season_beg_dt string,day_in_year_count
> smallint,day_in_qtr_count tinyint,day_in_month_count
> tinyint,day_in_week_count tinyint,rtl_year_beg_dt string,rtl_qtr_beg_dt
> string,rtl_month_beg_dt string,rtl_week_beg_dt string,cs_week_beg_dt
> string,cal_date string,day_of_week string,month_id string,prd_desc
> string,prd_flag string,prd_id string,prd_ind string,qtr_desc string,qtr_id
> string,qtr_ind string,retail_week string,retail_year string,retail_start_date
> string,retail_wk_end_date string,week_ind string,week_num_desc
> string,week_beg_date string,week_end_date string,week_in_year_id
> string,week_id string,week_beg_end_desc_mdy string,week_beg_end_desc_md
> string,year_id string,year_ind string,cal_dt_mns_1year_dt
> string,cal_dt_mns_2year_dt string,cal_dt_mns_1qtr_dt
> string,cal_dt_mns_2qtr_dt string,cal_dt_mns_1month_dt
> string,cal_dt_mns_2month_dt string,cal_dt_mns_1week_dt
> string,cal_dt_mns_2week_dt string,curr_cal_dt_mns_1year_yn_id
> tinyint,curr_cal_dt_mns_2year_yn_id tinyint,curr_cal_dt_mns_1qtr_yn_id
> tinyint,curr_cal_dt_mns_2qtr_yn_id tinyint,curr_cal_dt_mns_1month_yn_id
> tinyint,curr_cal_dt_mns_2month_yn_id tinyint,curr_cal_dt_mns_1week_yn_ind
> tinyint,curr_cal_dt_mns_2week_yn_ind tinyint,rtl_month_of_rtl_year_id
> string,rtl_qtr_of_rtl_year_id tinyint,rtl_week_of_rtl_year_id
> tinyint,season_of_year_id tinyint
[jira] [Commented] (KYLIN-4038) Lookup tables is empty when build on ORC tables.
[
https://issues.apache.org/jira/browse/KYLIN-4038?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16862825#comment-16862825
]
Alexander commented on KYLIN-4038:
--
Hello,
With external tables etherything is ok.
But in our business case, the data can be changed on 7 last days. To handle
this case, we try to use ACID tables, because they can be updated.
Our case is next:
# Load new data into hive.
# update data from (current date - 7 days) till now - that why ACID needed.
# Drop last partitions on Kylin cube. (last 7 days segments)
# Run buid cube for last 7 days.
If there is no way to use ACID, maybe any best practicies to handle this case?
> Lookup tables is empty when build on ORC tables.
>
>
> Key: KYLIN-4038
> URL: https://issues.apache.org/jira/browse/KYLIN-4038
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v2.6.2
> Environment: HDP 3 cluster.
>Reporter: Alexander
>Assignee: XiaoXiang Yu
>Priority: Blocker
> Attachments: image-2019-06-11-16-12-09-472.png
>
>
> Create tables, simple copy of learn_kylin project.
> {code:java}
> // Create tables
> create table kylin_sales_sida (trans_id bigint,part_dt date,lstg_format_name
> string,leaf_categ_id bigint,lstg_site_id int,slr_segment_cd smallint,price
> decimal(19,4),item_count bigint,seller_id bigint,buyer_id bigint,ops_user_id
> string,ops_region string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_sales_sida select * from kylin_sales;
> create table kylin_country_sida (country string,latitude double,longitude
> double,name string) stored as orc tblproperties ("orc.compress"="SNAPPY");
> insert into table kylin_country_sida select * from kylin_country;
> create table kylin_category_groupings_sida (leaf_categ_id
> bigint,leaf_categ_name string,site_id int,categ_busn_mgr
> string,categ_busn_unit string,regn_categ string,user_defined_field1
> string,user_defined_field3 string,kylin_groupings_cre_date
> string,kylin_groupings_upd_date string,kylin_groupings_cre_user
> string,kylin_groupings_upd_user string,meta_categ_id
> decimal(10,0),meta_categ_name string,categ_lvl2_id
> decimal(10,0),categ_lvl3_id decimal(10,0),categ_lvl4_id
> decimal(10,0),categ_lvl5_id decimal(10,0),categ_lvl6_id
> decimal(10,0),categ_lvl7_id decimal(10,0),categ_lvl2_name
> string,categ_lvl3_name string,categ_lvl4_name string,categ_lvl5_name
> string,categ_lvl6_name string,categ_lvl7_name string,categ_flags
> decimal(10,0),adult_categ_yn string,domain_id
> decimal(10,0),user_defined_field5 string,vcs_id decimal(10,0),gcs_id
> decimal(10,0),move_to decimal(10,0),sap_category_id decimal(10,0),src_id
> tinyint,bsns_vrtcl_name string) stored as orc tblproperties
> ("orc.compress"="SNAPPY");
> insert into table kylin_category_groupings_sida select * from
> kylin_category_groupings;
> create table kylin_cal_dt_sida (cal_dt date,year_beg_dt date,qtr_beg_dt
> date,month_beg_dt date,week_beg_dt date,age_for_year_id
> smallint,age_for_qtr_id smallint,age_for_month_id smallint,age_for_week_id
> smallint,age_for_dt_id smallint,age_for_rtl_year_id
> smallint,age_for_rtl_qtr_id smallint,age_for_rtl_month_id
> smallint,age_for_rtl_week_id smallint,age_for_cs_week_id
> smallint,day_of_cal_id int,day_of_year_id smallint,day_of_qtr_id
> smallint,day_of_month_id smallint,day_of_week_id int,week_of_year_id
> tinyint,week_of_cal_id int,month_of_qtr_id tinyint,month_of_year_id
> tinyint,month_of_cal_id smallint,qtr_of_year_id tinyint,qtr_of_cal_id
> smallint,year_of_cal_id smallint,year_end_dt string,qtr_end_dt
> string,month_end_dt string,week_end_dt string,cal_dt_name string,cal_dt_desc
> string,cal_dt_short_name string,ytd_yn_id tinyint,qtd_yn_id tinyint,mtd_yn_id
> tinyint,wtd_yn_id tinyint,season_beg_dt string,day_in_year_count
> smallint,day_in_qtr_count tinyint,day_in_month_count
> tinyint,day_in_week_count tinyint,rtl_year_beg_dt string,rtl_qtr_beg_dt
> string,rtl_month_beg_dt string,rtl_week_beg_dt string,cs_week_beg_dt
> string,cal_date string,day_of_week string,month_id string,prd_desc
> string,prd_flag string,prd_id string,prd_ind string,qtr_desc string,qtr_id
> string,qtr_ind string,retail_week string,retail_year string,retail_start_date
> string,retail_wk_end_date string,week_ind string,week_num_desc
> string,week_beg_date string,week_end_date string,week_in_year_id
> string,week_id string,week_beg_end_desc_mdy string,week_beg_end_desc_md
> string,year_id string,year_ind string,cal_dt_mns_1year_dt
> string,cal_dt_mns_2year_dt string,cal_dt_mns_1qtr_dt
> string,cal_dt_mns_2qtr_dt string,cal_dt_mns_1month_dt
> string,cal_dt_mns_2month_dt string,cal_dt_mns_1week_dt
> string,cal_dt_mns_2week_dt string,curr_cal_dt_mns_1year_yn_id
> tinyint,curr_cal_dt_mns_2
[jira] [Assigned] (KYLIN-3409) Write metric error when run a query.
[ https://issues.apache.org/jira/browse/KYLIN-3409?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander reassigned KYLIN-3409: Assignee: Alexander > Write metric error when run a query. > > > Key: KYLIN-3409 > URL: https://issues.apache.org/jira/browse/KYLIN-3409 > Project: Kylin > Issue Type: Bug >Reporter: praveenece >Assignee: Alexander >Priority: Major > > Hi, > I hit query(using sub query) against Cube the below exception is coming in > kylin log > but kylin UI response output. > why its coming? > 2018-06-13 17:15:51,153 WARN [Query 707b91e9-bcaa-46ef-be66-e7b74a3442d0-69] > service.QueryService:471 : Write metric error. > java.lang.NullPointerException > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetricsToReservoir(QueryMetricsFacade.java:125) > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetrics(QueryMetricsFacade.java:73) > at > org.apache.kylin.rest.service.QueryService.recordMetric(QueryService.java:560) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:469) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:390) > at > org.apache.kylin.rest.controller.QueryController.query(QueryController.java:86) > at sun.reflect.GeneratedMethodAccessor313.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:650) > at > org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Updated] (KYLIN-3409) Write metric error when run a query.
[ https://issues.apache.org/jira/browse/KYLIN-3409?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3409: - Attachment: 0001-KYLIN-3609-KYLIN-3409-KYLIN-4161.patch > Write metric error when run a query. > > > Key: KYLIN-3409 > URL: https://issues.apache.org/jira/browse/KYLIN-3409 > Project: Kylin > Issue Type: Bug >Reporter: praveenece >Assignee: Alexander >Priority: Major > Attachments: 0001-KYLIN-3609-KYLIN-3409-KYLIN-4161.patch > > > Hi, > I hit query(using sub query) against Cube the below exception is coming in > kylin log > but kylin UI response output. > why its coming? > 2018-06-13 17:15:51,153 WARN [Query 707b91e9-bcaa-46ef-be66-e7b74a3442d0-69] > service.QueryService:471 : Write metric error. > java.lang.NullPointerException > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetricsToReservoir(QueryMetricsFacade.java:125) > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetrics(QueryMetricsFacade.java:73) > at > org.apache.kylin.rest.service.QueryService.recordMetric(QueryService.java:560) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:469) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:390) > at > org.apache.kylin.rest.controller.QueryController.query(QueryController.java:86) > at sun.reflect.GeneratedMethodAccessor313.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:650) > at > org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Commented] (KYLIN-3409) Write metric error when run a query.
[ https://issues.apache.org/jira/browse/KYLIN-3409?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16974777#comment-16974777 ] Alexander commented on KYLIN-3409: -- Error occurs when queryType != OLAPQuery.EnumeratorTypeEnum.OLAP (i.e. LOOKUP_TABLE...). Handle this case. KYLIN-4161 - duplicate i think. > Write metric error when run a query. > > > Key: KYLIN-3409 > URL: https://issues.apache.org/jira/browse/KYLIN-3409 > Project: Kylin > Issue Type: Bug >Reporter: praveenece >Assignee: Alexander >Priority: Major > Attachments: 0001-KYLIN-3609-KYLIN-3409-KYLIN-4161.patch > > > Hi, > I hit query(using sub query) against Cube the below exception is coming in > kylin log > but kylin UI response output. > why its coming? > 2018-06-13 17:15:51,153 WARN [Query 707b91e9-bcaa-46ef-be66-e7b74a3442d0-69] > service.QueryService:471 : Write metric error. > java.lang.NullPointerException > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetricsToReservoir(QueryMetricsFacade.java:125) > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetrics(QueryMetricsFacade.java:73) > at > org.apache.kylin.rest.service.QueryService.recordMetric(QueryService.java:560) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:469) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:390) > at > org.apache.kylin.rest.controller.QueryController.query(QueryController.java:86) > at sun.reflect.GeneratedMethodAccessor313.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:650) > at > org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Updated] (KYLIN-3409) Write metric error when run a query.
[ https://issues.apache.org/jira/browse/KYLIN-3409?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3409: - Attachment: (was: 0001-KYLIN-3609-KYLIN-3409-KYLIN-4161.patch) > Write metric error when run a query. > > > Key: KYLIN-3409 > URL: https://issues.apache.org/jira/browse/KYLIN-3409 > Project: Kylin > Issue Type: Bug >Reporter: praveenece >Assignee: Alexander >Priority: Major > > Hi, > I hit query(using sub query) against Cube the below exception is coming in > kylin log > but kylin UI response output. > why its coming? > 2018-06-13 17:15:51,153 WARN [Query 707b91e9-bcaa-46ef-be66-e7b74a3442d0-69] > service.QueryService:471 : Write metric error. > java.lang.NullPointerException > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetricsToReservoir(QueryMetricsFacade.java:125) > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetrics(QueryMetricsFacade.java:73) > at > org.apache.kylin.rest.service.QueryService.recordMetric(QueryService.java:560) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:469) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:390) > at > org.apache.kylin.rest.controller.QueryController.query(QueryController.java:86) > at sun.reflect.GeneratedMethodAccessor313.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:650) > at > org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Updated] (KYLIN-3409) Write metric error when run a query.
[ https://issues.apache.org/jira/browse/KYLIN-3409?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander updated KYLIN-3409: - Attachment: 0001-KYLIN-3609-KYLIN-3409-KYLIN-4161-CubeSegmentStatisti.patch > Write metric error when run a query. > > > Key: KYLIN-3409 > URL: https://issues.apache.org/jira/browse/KYLIN-3409 > Project: Kylin > Issue Type: Bug >Reporter: praveenece >Assignee: Alexander >Priority: Major > Attachments: > 0001-KYLIN-3609-KYLIN-3409-KYLIN-4161-CubeSegmentStatisti.patch > > > Hi, > I hit query(using sub query) against Cube the below exception is coming in > kylin log > but kylin UI response output. > why its coming? > 2018-06-13 17:15:51,153 WARN [Query 707b91e9-bcaa-46ef-be66-e7b74a3442d0-69] > service.QueryService:471 : Write metric error. > java.lang.NullPointerException > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetricsToReservoir(QueryMetricsFacade.java:125) > at > org.apache.kylin.rest.metrics.QueryMetricsFacade.updateMetrics(QueryMetricsFacade.java:73) > at > org.apache.kylin.rest.service.QueryService.recordMetric(QueryService.java:560) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:469) > at > org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:390) > at > org.apache.kylin.rest.controller.QueryController.query(QueryController.java:86) > at sun.reflect.GeneratedMethodAccessor313.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) > at > org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) > at > org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) > at > org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) > at > org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) > at > org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967) > at > org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901) > at > org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) > at > org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:650) > at > org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) > at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) > at > org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Created] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
Alexander created KYLIN-4332:
Summary: Failed to load Hive Table java.lang.IllegalStateException
Key: KYLIN-4332
URL: https://issues.apache.org/jira/browse/KYLIN-4332
Project: Kylin
Issue Type: Bug
Affects Versions: v3.0.0-beta
Reporter: Alexander
Assignee: Alexander
When try to reload table metadata schema name is duplicated,
{code}
2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
controller.TableController:129 : Failed to load Hive Table2020-01-09
12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
Failed to load Hive Tablejava.lang.IllegalStateException at
com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
at
org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
at
org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Updated] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4332:
-
Description:
When try to reload table metadata schema name is duplicated.
(DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
{code:java}
2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
controller.TableController:129 : Failed to load Hive Table2020-01-09
12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
Failed to load Hive Tablejava.lang.IllegalStateException at
com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
at
org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
at
org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
{code}
was:
When try to reload table metadata schema name is duplicated,
{code}
2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
controller.TableController:129 : Failed to load Hive Table2020-01-09
12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
Failed to load Hive Tablejava.lang.IllegalStateException at
com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
at
org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
at
org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
{code}
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Commented] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17028739#comment-17028739
]
Alexander commented on KYLIN-4332:
--
Hello,
Version: Apache kylin 3.0.0-beta
Commit: f1bf247092ce0143f15ed859cc06a6a5ede39ed0
DataSource tables imported by TreeList button.
# Notice, what table added by tree have Schema and TableName both in child
elements. (KYLIN-4332-1.png)
# Try to Reload table or Unload table button (KYLIN-4332-1.png)
# Notice what schema printed twice, this will go while service invoke to
back-end. (KYLIN-4332-2.png)
# Click Sync button, and exception will be. (KYLIN-4332-3.png)
Returned message:
{code}
{
"code": "999",
"data": null,
"msg": null,
"stacktrace": "org.apache.kylin.rest.exception.InternalErrorException\n\tat
org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:130)\n\tat
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat
java.lang.reflect.Method.invoke(Method.java:498)\n\tat
org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)\n\tat
org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)\n\tat
org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)\n\tat
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)\n\tat
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)\n\tat
org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)\n\tat
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)\n\tat
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)\n\tat
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)\n\tat
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)\n\tat
javax.servlet.http.HttpServlet.service(HttpServlet.java:661)\n\tat
org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)\n\tat
javax.servlet.http.HttpServlet.service(HttpServlet.java:742)\n\tat
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)\n\tat
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)\n\tat
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)\n\tat
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)\n\tat
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:317)\n\tat
org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:127)\n\tat
org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:91)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:114)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:170)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilterInternal(BasicAuthenticationFilter.java:215)\n\tat
[jira] [Comment Edited] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17028739#comment-17028739
]
Alexander edited comment on KYLIN-4332 at 2/3/20 7:53 AM:
--
Hello,
Version: Apache kylin 3.0.0-beta
Commit: f1bf247092ce0143f15ed859cc06a6a5ede39ed0
DataSource tables imported by TreeList button.
# Notice, what table added by tree have Schema and TableName both in child
elements. (KYLIN-4332-1.png)
# Try to Reload table or Unload table button (KYLIN-4332-1.png)
# Notice what schema printed twice, this will go while service invoke to
back-end. (KYLIN-4332-2.png)
# Click Sync button, and exception will be. (KYLIN-4332-3.png)
Returned message:
{code:java}
{
"code": "999",
"data": null,
"msg": null,
"stacktrace": "org.apache.kylin.rest.exception.InternalErrorException\n\tat
org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:130)\n\tat
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat
java.lang.reflect.Method.invoke(Method.java:498)\n\tat
org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)\n\tat
org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)\n\tat
org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)\n\tat
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)\n\tat
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)\n\tat
org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)\n\tat
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)\n\tat
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)\n\tat
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)\n\tat
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)\n\tat
javax.servlet.http.HttpServlet.service(HttpServlet.java:661)\n\tat
org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)\n\tat
javax.servlet.http.HttpServlet.service(HttpServlet.java:742)\n\tat
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)\n\tat
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)\n\tat
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)\n\tat
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)\n\tat
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:317)\n\tat
org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:127)\n\tat
org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:91)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:114)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:170)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilterI
[jira] [Comment Edited] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17028739#comment-17028739
]
Alexander edited comment on KYLIN-4332 at 2/3/20 7:54 AM:
--
Hello,
Version: Apache kylin 3.0.0-beta
Commit: f1bf247092ce0143f15ed859cc06a6a5ede39ed0
DataSource tables imported by TreeList button.
# Notice, what table added by tree have Schema and TableName both in child
elements. (KYLIN-4332-1.png)
# Try to Reload table or Unload table button (KYLIN-4332-1.png)
# Notice what schema printed twice, this will go while service invoke to
back-end. (KYLIN-4332-2.png)
# Click Sync button, and exception will be. (KYLIN-4332-3.png)
Returned message:
{code:java}
{
"code": "999",
"data": null,
"msg": null,
"stacktrace": "org.apache.kylin.rest.exception.InternalErrorException\n\tat
org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:130)\n\tat
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat
java.lang.reflect.Method.invoke(Method.java:498)\n\tat
org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)\n\tat
org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)\n\tat
org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)\n\tat
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)\n\tat
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)\n\tat
org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)\n\tat
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)\n\tat
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)\n\tat
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)\n\tat
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)\n\tat
javax.servlet.http.HttpServlet.service(HttpServlet.java:661)\n\tat
org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)\n\tat
javax.servlet.http.HttpServlet.service(HttpServlet.java:742)\n\tat
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)\n\tat
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)\n\tat
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)\n\tat
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)\n\tat
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:317)\n\tat
org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:127)\n\tat
org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:91)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:114)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:170)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63)\n\tat
org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)\n\tat
org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilterI
[jira] [Updated] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4332:
-
Attachment: KYLIN-4332-1.png
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
> Attachments: KYLIN-4332-1.png, KYLIN-4332-2.png, KYLIN-4332-3.png
>
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Updated] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4332:
-
Attachment: KYLIN-4332-2.png
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
> Attachments: KYLIN-4332-1.png, KYLIN-4332-2.png, KYLIN-4332-3.png
>
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Updated] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4332:
-
Attachment: KYLIN-4332-3.png
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
> Attachments: KYLIN-4332-1.png, KYLIN-4332-2.png, KYLIN-4332-3.png
>
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Commented] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17029634#comment-17029634
]
Alexander commented on KYLIN-4332:
--
[~nichunen] Hello, add it again.
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
> Attachments: KYLIN-4332-1.png, KYLIN-4332-2.png, KYLIN-4332-3.png
>
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Comment Edited] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17029634#comment-17029634
]
Alexander edited comment on KYLIN-4332 at 2/4/20 8:19 AM:
--
[~nichunen] Hello, add it again. with success at rhis time )
was (Author: alex.sidorchuk85):
[~nichunen] Hello, add it again.
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
> Attachments: KYLIN-4332-1.png, KYLIN-4332-2.png, KYLIN-4332-3.png
>
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Commented] (KYLIN-4137) Accelerate metadata reloading
[ https://issues.apache.org/jira/browse/KYLIN-4137?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17030429#comment-17030429 ] Alexander commented on KYLIN-4137: -- Hello, this commit broke Convert Cuboid Data to HFile step in my v3.0.0-beta environment. I don't use mysql. Just add couple TopN measures, and COUNT_DISTINCT with more than one column (i.e. COUNT_DISTINCT seller_id, buyer_id). As result MeasureDesc -> FunctionDesc -> ParameterDesc -> PlainParameter serialization exception. If i make it (PlainParameter implements Serializable) serializable, then ContentReader.java T r = serializer.deserialize(din); deserialize without ParameterDesc.nextParameter wich causes error. Reverting this commit, solve problem for me. > Accelerate metadata reloading > - > > Key: KYLIN-4137 > URL: https://issues.apache.org/jira/browse/KYLIN-4137 > Project: Kylin > Issue Type: Improvement > Components: Metadata >Affects Versions: v2.6.2 >Reporter: Temple Zhou >Assignee: Temple Zhou >Priority: Minor > Fix For: v3.0.0-beta > > > Now, org.apache.kylin.metadata.cachesync.CachedCrudAssist#reloadAll is using > an inappropriate method to deal with MySQLJdbcMetadata. > As the every call of reloadAt(path) will access the database, which is almost > fine to HBase but really unfriendly to the MySQL(RDBMS). > I think we should get all of the resource with single request instead of the > separate request to get every resource. -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Reopened] (KYLIN-4137) Accelerate metadata reloading
[ https://issues.apache.org/jira/browse/KYLIN-4137?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Alexander reopened KYLIN-4137: -- > Accelerate metadata reloading > - > > Key: KYLIN-4137 > URL: https://issues.apache.org/jira/browse/KYLIN-4137 > Project: Kylin > Issue Type: Improvement > Components: Metadata >Affects Versions: v2.6.2 >Reporter: Temple Zhou >Assignee: Temple Zhou >Priority: Minor > Fix For: v3.0.0-beta > > > Now, org.apache.kylin.metadata.cachesync.CachedCrudAssist#reloadAll is using > an inappropriate method to deal with MySQLJdbcMetadata. > As the every call of reloadAt(path) will access the database, which is almost > fine to HBase but really unfriendly to the MySQL(RDBMS). > I think we should get all of the resource with single request instead of the > separate request to get every resource. -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Commented] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17030505#comment-17030505
]
Alexander commented on KYLIN-4332:
--
[~nichunen] Hello!
I'll check it again. Maybe this is affected by 3.1 build.
Thank you.
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
> Attachments: 2020-02-05 5.04.38.png, KYLIN-4332-1.png,
> KYLIN-4332-2.png, KYLIN-4332-3.png
>
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Closed] (KYLIN-4332) Failed to load Hive Table java.lang.IllegalStateException
[
https://issues.apache.org/jira/browse/KYLIN-4332?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander closed KYLIN-4332.
Resolution: Invalid
My environment issue.
> Failed to load Hive Table java.lang.IllegalStateException
> -
>
> Key: KYLIN-4332
> URL: https://issues.apache.org/jira/browse/KYLIN-4332
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.0.0-beta
>Reporter: Alexander
>Assignee: Alexander
>Priority: Minor
> Attachments: 2020-02-05 5.04.38.png, KYLIN-4332-1.png,
> KYLIN-4332-2.png, KYLIN-4332-3.png
>
>
> When try to reload table metadata schema name is duplicated.
> (DEFAULT.DEFAULT.KYLIN_SALES goes to TableController)
> {code:java}
> 2020-01-09 12:15:49,866 ERROR [http-nio-7070-exec-3]
> controller.TableController:129 : Failed to load Hive Table2020-01-09
> 12:15:49,866 ERROR [http-nio-7070-exec-3] controller.TableController:129 :
> Failed to load Hive Tablejava.lang.IllegalStateException at
> com.google.common.base.Preconditions.checkState(Preconditions.java:492) at
> org.apache.kylin.rest.service.TableService.extractHiveTableMeta(TableService.java:215)
> at
> org.apache.kylin.rest.service.TableService.loadHiveTablesToProject(TableService.java:138)
> at
> org.apache.kylin.rest.controller.TableController.loadHiveTables(TableController.java:113)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Created] (KYLIN-4803) Hive Global Dictionary - TopNMeasureType NullPointerException
Alexander created KYLIN-4803:
Summary: Hive Global Dictionary - TopNMeasureType
NullPointerException
Key: KYLIN-4803
URL: https://issues.apache.org/jira/browse/KYLIN-4803
Project: Kylin
Issue Type: Bug
Affects Versions: v3.1.1
Reporter: Alexander
I've read wiki article, and succesfully test it on count_distinct(bitmap)I've
read wiki article, and succesfully test it on count_distinct(bitmap).
When add measure like TOP_N, and have the same column.
i.e.
a. Measure BUYER_CNT_HLL - COUNT_DISTINCT Value:KYLIN_SALES.BUYER_ID,
Type:column bitmap
b. Measure TOP_SELLER - TOP_N SUM|ORDER BY:KYLIN_SALES.PRICE Group
By:KYLIN_SALES.BUYER_ID topn(100,4)
Then exception on step - Build Cube with Spark araise:
{code:java}
20/10/30 07:33:11 ERROR executor.Executor: Exception in task 1.0 in stage 0.0
(TID 1)20/10/30 07:33:11 ERROR executor.Executor: Exception in task 1.0 in
stage 0.0 (TID 1)java.lang.NullPointerException at
org.apache.kylin.dimension.DictionaryDimEnc.(DictionaryDimEnc.java:63) at
org.apache.kylin.dimension.DictionaryDimEnc.(DictionaryDimEnc.java:58) at
org.apache.kylin.measure.topn.TopNMeasureType.getDimensionEncodings(TopNMeasureType.java:535)
at
org.apache.kylin.measure.topn.TopNMeasureType.access$200(TopNMeasureType.java:58)
at
org.apache.kylin.measure.topn.TopNMeasureType$1.valueOf(TopNMeasureType.java:146)
at
org.apache.kylin.measure.topn.TopNMeasureType$1.valueOf(TopNMeasureType.java:127)
at
org.apache.kylin.engine.mr.common.BaseCuboidBuilder.buildValueObjects(BaseCuboidBuilder.java:118)
{code}
It's because there is no dictionary for BUYER_ID
In log file i see:
Enable hive global dictionary for DEFAULT.KYLIN_SALES.BUYER_ID
But in fact, TopNMeasureType: dictionaryMap do not have
DEFAULT.KYLIN_SALES.BUYER_ID
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Updated] (KYLIN-4803) Hive Global Dictionary - TopNMeasureType NullPointerException
[
https://issues.apache.org/jira/browse/KYLIN-4803?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4803:
-
Attachment: spark_executor.log
> Hive Global Dictionary - TopNMeasureType NullPointerException
> -
>
> Key: KYLIN-4803
> URL: https://issues.apache.org/jira/browse/KYLIN-4803
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.1.1
>Reporter: Alexander
>Priority: Major
> Attachments: spark_executor.log
>
>
> I've read wiki article, and succesfully test it on count_distinct(bitmap)I've
> read wiki article, and succesfully test it on count_distinct(bitmap).
> When add measure like TOP_N, and have the same column.
> i.e.
> a. Measure BUYER_CNT_HLL - COUNT_DISTINCT Value:KYLIN_SALES.BUYER_ID,
> Type:column bitmap
> b. Measure TOP_SELLER - TOP_N SUM|ORDER BY:KYLIN_SALES.PRICE Group
> By:KYLIN_SALES.BUYER_ID topn(100,4)
> Then exception on step - Build Cube with Spark araise:
> {code:java}
> 20/10/30 07:33:11 ERROR executor.Executor: Exception in task 1.0 in stage 0.0
> (TID 1)20/10/30 07:33:11 ERROR executor.Executor: Exception in task 1.0 in
> stage 0.0 (TID 1)java.lang.NullPointerException at
> org.apache.kylin.dimension.DictionaryDimEnc.(DictionaryDimEnc.java:63)
> at
> org.apache.kylin.dimension.DictionaryDimEnc.(DictionaryDimEnc.java:58)
> at
> org.apache.kylin.measure.topn.TopNMeasureType.getDimensionEncodings(TopNMeasureType.java:535)
> at
> org.apache.kylin.measure.topn.TopNMeasureType.access$200(TopNMeasureType.java:58)
> at
> org.apache.kylin.measure.topn.TopNMeasureType$1.valueOf(TopNMeasureType.java:146)
> at
> org.apache.kylin.measure.topn.TopNMeasureType$1.valueOf(TopNMeasureType.java:127)
> at
> org.apache.kylin.engine.mr.common.BaseCuboidBuilder.buildValueObjects(BaseCuboidBuilder.java:118)
> {code}
> It's because there is no dictionary for BUYER_ID
> In log file i see:
> Enable hive global dictionary for DEFAULT.KYLIN_SALES.BUYER_ID
> But in fact, TopNMeasureType: dictionaryMap do not have
> DEFAULT.KYLIN_SALES.BUYER_ID
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Commented] (KYLIN-4803) Hive Global Dictionary - TopNMeasureType NullPointerException
[
https://issues.apache.org/jira/browse/KYLIN-4803?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17223594#comment-17223594
]
Alexander commented on KYLIN-4803:
--
Is it problem with TopNMeasureType? Or in this case, dictionary must be both in
MR Hive and old style?
> Hive Global Dictionary - TopNMeasureType NullPointerException
> -
>
> Key: KYLIN-4803
> URL: https://issues.apache.org/jira/browse/KYLIN-4803
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.1.1
>Reporter: Alexander
>Priority: Major
> Attachments: spark_executor.log
>
>
> I've read wiki article, and succesfully test it on count_distinct(bitmap)I've
> read wiki article, and succesfully test it on count_distinct(bitmap).
> When add measure like TOP_N, and have the same column.
> i.e.
> a. Measure BUYER_CNT_HLL - COUNT_DISTINCT Value:KYLIN_SALES.BUYER_ID,
> Type:column bitmap
> b. Measure TOP_SELLER - TOP_N SUM|ORDER BY:KYLIN_SALES.PRICE Group
> By:KYLIN_SALES.BUYER_ID topn(100,4)
> Then exception on step - Build Cube with Spark araise:
> {code:java}
> 20/10/30 07:33:11 ERROR executor.Executor: Exception in task 1.0 in stage 0.0
> (TID 1)20/10/30 07:33:11 ERROR executor.Executor: Exception in task 1.0 in
> stage 0.0 (TID 1)java.lang.NullPointerException at
> org.apache.kylin.dimension.DictionaryDimEnc.(DictionaryDimEnc.java:63)
> at
> org.apache.kylin.dimension.DictionaryDimEnc.(DictionaryDimEnc.java:58)
> at
> org.apache.kylin.measure.topn.TopNMeasureType.getDimensionEncodings(TopNMeasureType.java:535)
> at
> org.apache.kylin.measure.topn.TopNMeasureType.access$200(TopNMeasureType.java:58)
> at
> org.apache.kylin.measure.topn.TopNMeasureType$1.valueOf(TopNMeasureType.java:146)
> at
> org.apache.kylin.measure.topn.TopNMeasureType$1.valueOf(TopNMeasureType.java:127)
> at
> org.apache.kylin.engine.mr.common.BaseCuboidBuilder.buildValueObjects(BaseCuboidBuilder.java:118)
> {code}
> It's because there is no dictionary for BUYER_ID
> In log file i see:
> Enable hive global dictionary for DEFAULT.KYLIN_SALES.BUYER_ID
> But in fact, TopNMeasureType: dictionaryMap do not have
> DEFAULT.KYLIN_SALES.BUYER_ID
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Updated] (KYLIN-4804) Hive Global Dictionary - with measure (PRICE) NumberFormatException
[
https://issues.apache.org/jira/browse/KYLIN-4804?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-4804:
-
Attachment: spark_executor.log
> Hive Global Dictionary - with measure (PRICE) NumberFormatException
> ---
>
> Key: KYLIN-4804
> URL: https://issues.apache.org/jira/browse/KYLIN-4804
> Project: Kylin
> Issue Type: Bug
>Affects Versions: v3.1.1
>Reporter: Alexander
>Priority: Major
> Attachments: spark_executor.log
>
>
> If i try to use Global dict with measure (PRICE)
> Cubing with spark - exception
> {code:java}
> java.lang.NumberFormatException: For input string:
> "7623."java.lang.NumberFormatException: For input string: "7623." at
> java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
> at java.lang.Integer.parseInt(Integer.java:580) at
> java.lang.Integer.parseInt(Integer.java:615) at
> org.apache.kylin.measure.bitmap.BitmapMeasureType$1.valueOf(BitmapMeasureType.java:118)
> at
> org.apache.kylin.measure.bitmap.BitmapMeasureType$1.valueOf(BitmapMeasureType.java:99)
> at
> org.apache.kylin.engine.mr.common.BaseCuboidBuilder.buildValueObjects(BaseCuboidBuilder.java:116)
> at
> org.apache.kylin.engine.spark.SparkCubingByLayer$EncodeBaseCuboid.call(SparkCubingByLayer.java:297)
> at
> org.apache.kylin.engine.spark.SparkCubingByLayer$EncodeBaseCuboid.call(SparkCubingByLayer.java:257)
> {code}
> it's because
> {code:java}
>// In order to keep compatibility with old version, tinyint/smallint/int
> column use value directly, without dictionary
> private boolean needDictionaryColumn(FunctionDesc functionDesc) {
> DataType dataType =
> functionDesc.getParameter().getColRefs().get(0).getType();
> if (functionDesc.isMrDict()) {
> // If isMrDict set to true, it means related column has been
> // encoded in previous step by Hive Global Dictionary
> return false;
> }
> if (dataType.isIntegerFamily() && !dataType.isBigInt()) {
> return false;
> }
> return true;
> }
> {code}
> returns false due to isMrDict is true in this case and it goes id =
> Integer.parseInt(values[0]);
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Created] (KYLIN-4804) Hive Global Dictionary - with measure (PRICE) NumberFormatException
Alexander created KYLIN-4804:
Summary: Hive Global Dictionary - with measure (PRICE)
NumberFormatException
Key: KYLIN-4804
URL: https://issues.apache.org/jira/browse/KYLIN-4804
Project: Kylin
Issue Type: Bug
Affects Versions: v3.1.1
Reporter: Alexander
Attachments: spark_executor.log
If i try to use Global dict with measure (PRICE)
Cubing with spark - exception
{code:java}
java.lang.NumberFormatException: For input string:
"7623."java.lang.NumberFormatException: For input string: "7623." at
java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
at java.lang.Integer.parseInt(Integer.java:580) at
java.lang.Integer.parseInt(Integer.java:615) at
org.apache.kylin.measure.bitmap.BitmapMeasureType$1.valueOf(BitmapMeasureType.java:118)
at
org.apache.kylin.measure.bitmap.BitmapMeasureType$1.valueOf(BitmapMeasureType.java:99)
at
org.apache.kylin.engine.mr.common.BaseCuboidBuilder.buildValueObjects(BaseCuboidBuilder.java:116)
at
org.apache.kylin.engine.spark.SparkCubingByLayer$EncodeBaseCuboid.call(SparkCubingByLayer.java:297)
at
org.apache.kylin.engine.spark.SparkCubingByLayer$EncodeBaseCuboid.call(SparkCubingByLayer.java:257)
{code}
it's because
{code:java}
// In order to keep compatibility with old version, tinyint/smallint/int
column use value directly, without dictionary
private boolean needDictionaryColumn(FunctionDesc functionDesc) {
DataType dataType =
functionDesc.getParameter().getColRefs().get(0).getType();
if (functionDesc.isMrDict()) {
// If isMrDict set to true, it means related column has been
// encoded in previous step by Hive Global Dictionary
return false;
}
if (dataType.isIntegerFamily() && !dataType.isBigInt()) {
return false;
}
return true;
}
{code}
returns false due to isMrDict is true in this case and it goes id =
Integer.parseInt(values[0]);
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Created] (KYLIN-4805) Hive Global Dictionary - COUNT_DISTINCT(HLL) with multiple parameters
Alexander created KYLIN-4805:
Summary: Hive Global Dictionary - COUNT_DISTINCT(HLL) with
multiple parameters
Key: KYLIN-4805
URL: https://issues.apache.org/jira/browse/KYLIN-4805
Project: Kylin
Issue Type: Bug
Affects Versions: v3.1.1
Reporter: Alexander
When i try to use COUNT_DISTINCT(HLL) with multiple parameters.
I see in code BaseCuboidBuilder.checkHiveGlobalDictionaryColumn
{code:java}
for (MeasureDesc measure : measureDescList) {
if
(measure.getFunction().getExpression().equalsIgnoreCase(FunctionDesc.FUNC_COUNT_DISTINCT))
{
FunctionDesc functionDesc = measure.getFunction();
TblColRef colRef =
functionDesc.getParameter().getColRefs().get(0);
if (mrDictColumnSet.contains(JoinedFlatTable.colName(colRef,
true))) {
functionDesc.setMrDict(true);
logger.info("Enable hive global dictionary for {}", colRef);
measure.setFunction(functionDesc);
}
}
}
{code}
As i see, here supported only BITMAP with one parameter.
Created:
|SELLER_CNT_HLL|COUNT_DISTINCT| * Value:*KYLIN_SALES.SELLER_ID*, Type:*column*
* Value:*KYLIN_SALES.BUYER_ID*, Type:*column*|hllc(10)|
*kylin.dictionary.mr-hive.columns*
KYLIN_SALES_SELLER_ID
Run build cube: Ok
SQL: SELECT COUNT(DISTINCT seller_id, buyer_id) AS DIST_SELLER FROM kylin_sales;
Hive:
{code:java}
+--+
| dist_seller |
+--+
| 9998 |
+--+
1 row selected (22.947 seconds)
{code}
Kylin:
{code:java}
Results (1)
DIST_SELLER
9983
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Created] (KYLIN-4974) Kylin does not handle fucntion CURRENT_TIME().
Alexander created KYLIN-4974:
Summary: Kylin does not handle fucntion CURRENT_TIME().
Key: KYLIN-4974
URL: https://issues.apache.org/jira/browse/KYLIN-4974
Project: Kylin
Issue Type: Bug
Components: Query Engine
Affects Versions: v3.1.1
Reporter: Alexander
Assignee: Alexander
Fix For: v3.1.2
Select query:
{code:java}
==[QUERY]===
Query Id: 4e8c6498-9736-604f-ca84-c537e749448b
SQL: select CURRENT_TIME() from KYLIN_SALES
{code}
Response:
{code:java}
org.apache.kylin.rest.exception.InternalErrorException: No DataTypeSerializer
for type timeorg.apache.kylin.rest.exception.InternalErrorException: No
DataTypeSerializer for type timewhile executing SQL: "select * from (select
CURRENT_TIME() from KYLIN_SALES) limit 5" at
org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:486)
at
org.apache.kylin.rest.service.QueryService.doQueryWithCache(QueryService.java:401)
at
org.apache.kylin.rest.controller.QueryController.query(QueryController.java:93)
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Created] (KYLIN-5066) CubeMigrationCrossClusterCLI - distcp (java.io.FileNotFoundException): Path is not a file
Alexander created KYLIN-5066:
Summary: CubeMigrationCrossClusterCLI - distcp
(java.io.FileNotFoundException): Path is not a file
Key: KYLIN-5066
URL: https://issues.apache.org/jira/browse/KYLIN-5066
Project: Kylin
Issue Type: Bug
Components: Client - CLI
Affects Versions: v3.1.2
Environment: HDP 3.1.4
Reporter: Alexander
Assignee: Alexander
Fix For: v3.1.3
Try to migrate cube between clusters:
{code:java}
$KYLIN_HOME/bin/kylin.sh
org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI -kylinUriSrc
ADMIN:KYLIN@hostname1:7070 -kylinUriDst ADMIN:KYLIN@hostname12:7070 -cube
CUBE_NAME -execute true -schemaOnly false -overwrite true
{code}
Got exception:
{code:java}
2021-08-20 00:07:04,533 ERROR [Cube-LOY_TML_FAVCAT_V1_4-data-migration-pool-7]
migration.CubeMigrationCrossClusterCLI:679 : fail to migrate htable
KYLIN_7E0QA66PIM due to {} 2021-08-20 00:07:04,533 ERROR
[Cube-LOY_TML_FAVCAT_V1_4-data-migration-pool-7]
migration.CubeMigrationCrossClusterCLI:679 : fail to migrate htable
KYLIN_7E0QA66PIM due to {} java.io.FileNotFoundException: Path is not a file:
/apps/hbase/data/data/default/KYLIN_7E0QA66PIM at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:90) at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:76) at
org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.getBlockLocations(FSDirStatAndListingOp.java:158)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1931)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:738)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:426)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:524)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1025) at
org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:876) at
org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:822) at
java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2682)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88)
at org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:864)
at org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:851) at
org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:840) at
org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1004) at
org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:320)
at
org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:316)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:328)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:899) at
org.apache.hadoop.tools.FileBasedCopyListing.fetchFileList(FileBasedCopyListing.java:78)
at
org.apache.hadoop.tools.FileBasedCopyListing.doBuildListing(FileBasedCopyListing.java:69)
at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86) at
org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:368) at
org.apache.hadoop.tools.DistCp.prepareFileListing(DistCp.java:96) at
org.apache.hadoop.tools.DistCp.createAndSubmitJob(DistCp.java:205) at
org.apache.hadoop.tools.DistCp.execute(DistCp.java:182) at
org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI.copyHDFSPath(CubeMigrationCrossClusterCLI.java:699)
at
org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI.copyHFileByDistCp(CubeMigrationCrossClusterCLI.java:689)
at
org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI.copyHTable(CubeMigrationCrossClusterCLI.java:661)
at
org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI.copyHTable(CubeMigrationCrossClusterCLI.java:613)
at
org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI.access$200(CubeMigrationCrossClusterCLI.java:87)
[jira] [Updated] (KYLIN-5066) CubeMigrationCrossClusterCLI - distcp (java.io.FileNotFoundException): Path is not a file
[
https://issues.apache.org/jira/browse/KYLIN-5066?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexander updated KYLIN-5066:
-
Attachment: 0001-KYLIN-5066-CubeMigrationCrossClusterCLI-distcp-java..patch
> CubeMigrationCrossClusterCLI - distcp (java.io.FileNotFoundException): Path
> is not a file
> --
>
> Key: KYLIN-5066
> URL: https://issues.apache.org/jira/browse/KYLIN-5066
> Project: Kylin
> Issue Type: Bug
> Components: Client - CLI
>Affects Versions: v3.1.2
> Environment: HDP 3.1.4
>Reporter: Alexander
>Assignee: Alexander
>Priority: Major
> Fix For: v3.1.3
>
> Attachments:
> 0001-KYLIN-5066-CubeMigrationCrossClusterCLI-distcp-java..patch
>
>
> Try to migrate cube between clusters:
> {code:java}
> $KYLIN_HOME/bin/kylin.sh
> org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI -kylinUriSrc
> ADMIN:KYLIN@hostname1:7070 -kylinUriDst ADMIN:KYLIN@hostname12:7070 -cube
> CUBE_NAME -execute true -schemaOnly false -overwrite true
> {code}
> Got exception:
> {code:java}
> 2021-08-20 00:07:04,533 ERROR
> [Cube-LOY_TML_FAVCAT_V1_4-data-migration-pool-7]
> migration.CubeMigrationCrossClusterCLI:679 : fail to migrate htable
> KYLIN_7E0QA66PIM due to {} 2021-08-20 00:07:04,533 ERROR
> [Cube-LOY_TML_FAVCAT_V1_4-data-migration-pool-7]
> migration.CubeMigrationCrossClusterCLI:679 : fail to migrate htable
> KYLIN_7E0QA66PIM due to {} java.io.FileNotFoundException: Path is not a file:
> /apps/hbase/data/data/default/KYLIN_7E0QA66PIM at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:90)
> at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:76)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.getBlockLocations(FSDirStatAndListingOp.java:158)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1931)
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:738)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:426)
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:524)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1025) at
> org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:876) at
> org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:822) at
> java.security.AccessController.doPrivileged(Native Method) at
> javax.security.auth.Subject.doAs(Subject.java:422) at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2682)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at
> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121)
> at
> org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88)
> at
> org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:864) at
> org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:851) at
> org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:840) at
> org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1004) at
> org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:320)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:316)
> at
> org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:328)
> at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:899) at
> org.apache.hadoop.tools.FileBasedCopyListing.fetchFileList(FileBasedCopyListing.java:78)
> at
> org.apache.hadoop.tools.FileBasedCopyListing.doBuildListing(FileBasedCopyListing.java:69)
> at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86) at
> org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:368) at
> org.apache.hadoop.tools.DistCp.prepareFileListing(DistCp.java:96) at
> org.apache.hadoop.tools.DistCp.createAndSubmitJob(DistCp.java:205) at
> org.apache.hadoop.tools.DistCp.execute(DistCp.java:182) at
> org.apache.kylin.tool.mi
[jira] [Commented] (KYLIN-5066) CubeMigrationCrossClusterCLI - distcp (java.io.FileNotFoundException): Path is not a file
[
https://issues.apache.org/jira/browse/KYLIN-5066?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17402027#comment-17402027
]
Alexander commented on KYLIN-5066:
--
Hello,
patch in attachment fixes issue on Hadoop3, that came in Commit
57cced6ba8b683a7db6239a0d042734c171aba44
In Kylin3 branch there is no such commit, and i don't undestand where to merge
this...
> CubeMigrationCrossClusterCLI - distcp (java.io.FileNotFoundException): Path
> is not a file
> --
>
> Key: KYLIN-5066
> URL: https://issues.apache.org/jira/browse/KYLIN-5066
> Project: Kylin
> Issue Type: Bug
> Components: Client - CLI
>Affects Versions: v3.1.2
> Environment: HDP 3.1.4
>Reporter: Alexander
>Assignee: Alexander
>Priority: Major
> Fix For: v3.1.3
>
> Attachments:
> 0001-KYLIN-5066-CubeMigrationCrossClusterCLI-distcp-java..patch
>
>
> Try to migrate cube between clusters:
> {code:java}
> $KYLIN_HOME/bin/kylin.sh
> org.apache.kylin.tool.migration.CubeMigrationCrossClusterCLI -kylinUriSrc
> ADMIN:KYLIN@hostname1:7070 -kylinUriDst ADMIN:KYLIN@hostname12:7070 -cube
> CUBE_NAME -execute true -schemaOnly false -overwrite true
> {code}
> Got exception:
> {code:java}
> 2021-08-20 00:07:04,533 ERROR
> [Cube-LOY_TML_FAVCAT_V1_4-data-migration-pool-7]
> migration.CubeMigrationCrossClusterCLI:679 : fail to migrate htable
> KYLIN_7E0QA66PIM due to {} 2021-08-20 00:07:04,533 ERROR
> [Cube-LOY_TML_FAVCAT_V1_4-data-migration-pool-7]
> migration.CubeMigrationCrossClusterCLI:679 : fail to migrate htable
> KYLIN_7E0QA66PIM due to {} java.io.FileNotFoundException: Path is not a file:
> /apps/hbase/data/data/default/KYLIN_7E0QA66PIM at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:90)
> at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:76)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.getBlockLocations(FSDirStatAndListingOp.java:158)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1931)
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:738)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:426)
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:524)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1025) at
> org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:876) at
> org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:822) at
> java.security.AccessController.doPrivileged(Native Method) at
> javax.security.auth.Subject.doAs(Subject.java:422) at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2682)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at
> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121)
> at
> org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88)
> at
> org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:864) at
> org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:851) at
> org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:840) at
> org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1004) at
> org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:320)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:316)
> at
> org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:328)
> at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:899) at
> org.apache.hadoop.tools.FileBasedCopyListing.fetchFileList(FileBasedCopyListing.java:78)
> at
> org.apache.hadoop.tools.FileBasedCopyListing.doBuildListing(FileBasedCopyListing.java:69)
> at org.apache.hadoop.tools.CopyListing.buildListing(CopyListing.java:86) at
> org.apache.hadoop.tools.DistCp.createInputFileListing(DistCp.java:368) at
> org.apache.hadoop.tools.DistCp.prepareFileListi
