[ https://issues.apache.org/jira/browse/KYLIN-1568?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Xingxing Di updated KYLIN-1568: ------------------------------- Description: I looked into the code , found this : valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); It looks like the ROWVALUE_BUFFER_SIZE is smaller than the value bytes length. The exeption is : 2016-04-07 10:18:57,311 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : java.nio.BufferOverflowException at java.nio.HeapByteBuffer.put(HeapByteBuffer.java:200) at org.apache.kylin.measure.bitmap.BitmapCounter.writeRegisters(BitmapCounter.java:85) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:40) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:30) at org.apache.kylin.measure.MeasureCodec.encode(MeasureCodec.java:89) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:105) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:43) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162) The cube info : { "uuid": "6990333d-ef59-44e3-b46a-7e571d002698", "version": "1.5.0", "name": "flw_test2_cube_1", "description": "", "dimensions": [ { "name": "TMP.TMP_FLW_MAIN_2.SITE_CLASS", "table": "TMP.TMP_FLW_MAIN_2", "column": "SITE_CLASS", "derived": null }, { "name": "TMP.TMP_FLW_MAIN_2.DEVICE_ID", "table": "TMP.TMP_FLW_MAIN_2", "column": "DEVICE_ID", "derived": null }, { "name": "TMP.TMP_FLW_MAIN_2.REF_DOMAIN", "table": "TMP.TMP_FLW_MAIN_2", "column": "REF_DOMAIN", "derived": null }, { "name": "TMP.TMP_FLW_MAIN_2.NEW_FLAG", "table": "TMP.TMP_FLW_MAIN_2", "column": "NEW_FLAG", "derived": null }, { "name": "TMP.TMP_FLW_MAIN_2.DEALER_ID", "table": "TMP.TMP_FLW_MAIN_2", "column": "DEALER_ID", "derived": null }, { "name": "TMP.TMP_FLW_MAIN_2.KEY_WORD", "table": "TMP.TMP_FLW_MAIN_2", "column": "KEY_WORD", "derived": null }, { "name": "TMP.OLAP_DIM_CITY_ALL_DERIVED", "table": "TMP.OLAP_DIM_CITY_ALL", "column": null, "derived": [ "ID", "CITY_ID", "CITY_NAME", "PROVINCE_ID", "PROVINCE_NAME" ] }, { "name": "TMP.OLAP_DIM_SITE_ALL_DERIVED", "table": "TMP.OLAP_DIM_SITE_ALL", "column": null, "derived": [ "ID", "SITE_ID", "SITE_NAME", "CATEGORY_ID", "CATEGORY_NAME", "SUB_CATEGORY_ID", "SUB_CATEGORY_NAME" ] }, { "name": "OLAP.OLAP_DIM_DAY_DERIVED", "table": "OLAP.OLAP_DIM_DAY", "column": null, "derived": [ "DIM_DAY_ID", "DAY_DATE", "WEEKDAY_ID", "DIM_MONTH_ID" ] }, { "name": "TMP.OLAP_DIM_SPEC_ALL_DERIVED", "table": "TMP.OLAP_DIM_SPEC_ALL", "column": null, "derived": [ "ID", "SPEC_ID", "SPEC_NAME", "SERIES_ID", "SERIES_NAME", "BRAND_ID", "BRAND_NAME", "LEVEL_ID", "LEVEL_NAME", "FACTORY_ID", "FACTORY_NAME" ] } ], "measures": [ { "name": "_COUNT_", "function": { "expression": "COUNT", "parameter": { "type": "constant", "value": "1", "next_parameter": null }, "returntype": "bigint" }, "dependent_measure_ref": null }, { "name": "PV", "function": { "expression": "SUM", "parameter": { "type": "column", "value": "PV", "next_parameter": null }, "returntype": "bigint" }, "dependent_measure_ref": null }, { "name": "USER_ID", "function": { "expression": "COUNT_DISTINCT", "parameter": { "type": "column", "value": "USER_ID", "next_parameter": null }, "returntype": "bitmap" }, "dependent_measure_ref": null } ], "rowkey": { "rowkey_columns": [ { "column": "DT", "encoding": "dict" }, { "column": "SPEC_ID", "encoding": "dict" }, { "column": "SITE_ID", "encoding": "dict" }, { "column": "DEALER_ID", "encoding": "dict" }, { "column": "CITY_ID", "encoding": "dict" }, { "column": "KEY_WORD", "encoding": "dict" }, { "column": "REF_DOMAIN", "encoding": "dict" }, { "column": "SITE_CLASS", "encoding": "dict" }, { "column": "DEVICE_ID", "encoding": "dict" }, { "column": "NEW_FLAG", "encoding": "dict" } ] }, "signature": "8O48Zjj6YujqabUmbVcg+A==", "last_modified": 1459996931395, "model_name": "flw_test2", "null_string": null, "hbase_mapping": { "column_family": [ { "name": "F1", "columns": [ { "qualifier": "M", "measure_refs": [ "_COUNT_", "PV" ] } ] }, { "name": "F2", "columns": [ { "qualifier": "M", "measure_refs": [ "USER_ID" ] } ] } ] }, "aggregation_groups": [ { "includes": [ "SPEC_ID", "CITY_ID", "DT", "SITE_CLASS", "SITE_ID" ], "select_rule": { "hierarchy_dims": [], "mandatory_dims": [ "DT" ], "joint_dims": [ [ "SITE_CLASS", "CITY_ID", "SPEC_ID" ] ] } }, { "includes": [ "DT", "SITE_CLASS", "DEVICE_ID", "REF_DOMAIN", "SITE_ID" ], "select_rule": { "hierarchy_dims": [], "mandatory_dims": [ "DT" ], "joint_dims": [ [ "DEVICE_ID", "REF_DOMAIN", "SITE_ID" ] ] } }, { "includes": [ "DT", "NEW_FLAG", "SPEC_ID", "REF_DOMAIN", "DEALER_ID" ], "select_rule": { "hierarchy_dims": [], "mandatory_dims": [ "DT" ], "joint_dims": [] } }, { "includes": [ "DT", "KEY_WORD", "SITE_ID", "DEVICE_ID", "REF_DOMAIN" ], "select_rule": { "hierarchy_dims": [], "mandatory_dims": [ "DT" ], "joint_dims": [] } } ], "notify_list": [], "status_need_notify": [], "partition_date_start": 1459814400000, "partition_date_end": 3153600000000, "auto_merge_time_ranges": [ 604800000, 2419200000 ], "retention_range": 0, "engine_type": 2, "storage_type": 2, "override_kylin_properties": {} } > Got java.nio.BufferOverflowException when Build Base Cuboid Data > ----------------------------------------------------------------- > > Key: KYLIN-1568 > URL: https://issues.apache.org/jira/browse/KYLIN-1568 > Project: Kylin > Issue Type: Bug > Components: Job Engine > Affects Versions: v1.5.0 > Environment: hadoop2.4.1 , hive1.0 , hbase0.98 > Reporter: Xingxing Di > Assignee: Dong Li > > I looked into the code , found this : > valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); > It looks like the ROWVALUE_BUFFER_SIZE is smaller than the value bytes length. > The exeption is : > 2016-04-07 10:18:57,311 WARN [main] org.apache.hadoop.mapred.YarnChild: > Exception running child : java.nio.BufferOverflowException at > java.nio.HeapByteBuffer.put(HeapByteBuffer.java:200) at > org.apache.kylin.measure.bitmap.BitmapCounter.writeRegisters(BitmapCounter.java:85) > at > org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:40) > at > org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:30) > at > org.apache.kylin.measure.MeasureCodec.encode(MeasureCodec.java:89) at > org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:105) > at > org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:43) > at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) > at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) > at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) > at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167) at > java.security.AccessController.doPrivileged(Native Method) at > javax.security.auth.Subject.doAs(Subject.java:415) at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) > at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162) > The cube info : > { > "uuid": "6990333d-ef59-44e3-b46a-7e571d002698", > "version": "1.5.0", > "name": "flw_test2_cube_1", > "description": "", > "dimensions": [ > { > "name": "TMP.TMP_FLW_MAIN_2.SITE_CLASS", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "SITE_CLASS", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.DEVICE_ID", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "DEVICE_ID", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.REF_DOMAIN", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "REF_DOMAIN", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.NEW_FLAG", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "NEW_FLAG", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.DEALER_ID", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "DEALER_ID", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.KEY_WORD", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "KEY_WORD", > "derived": null > }, > { > "name": "TMP.OLAP_DIM_CITY_ALL_DERIVED", > "table": "TMP.OLAP_DIM_CITY_ALL", > "column": null, > "derived": [ > "ID", > "CITY_ID", > "CITY_NAME", > "PROVINCE_ID", > "PROVINCE_NAME" > ] > }, > { > "name": "TMP.OLAP_DIM_SITE_ALL_DERIVED", > "table": "TMP.OLAP_DIM_SITE_ALL", > "column": null, > "derived": [ > "ID", > "SITE_ID", > "SITE_NAME", > "CATEGORY_ID", > "CATEGORY_NAME", > "SUB_CATEGORY_ID", > "SUB_CATEGORY_NAME" > ] > }, > { > "name": "OLAP.OLAP_DIM_DAY_DERIVED", > "table": "OLAP.OLAP_DIM_DAY", > "column": null, > "derived": [ > "DIM_DAY_ID", > "DAY_DATE", > "WEEKDAY_ID", > "DIM_MONTH_ID" > ] > }, > { > "name": "TMP.OLAP_DIM_SPEC_ALL_DERIVED", > "table": "TMP.OLAP_DIM_SPEC_ALL", > "column": null, > "derived": [ > "ID", > "SPEC_ID", > "SPEC_NAME", > "SERIES_ID", > "SERIES_NAME", > "BRAND_ID", > "BRAND_NAME", > "LEVEL_ID", > "LEVEL_NAME", > "FACTORY_ID", > "FACTORY_NAME" > ] > } > ], > "measures": [ > { > "name": "_COUNT_", > "function": { > "expression": "COUNT", > "parameter": { > "type": "constant", > "value": "1", > "next_parameter": null > }, > "returntype": "bigint" > }, > "dependent_measure_ref": null > }, > { > "name": "PV", > "function": { > "expression": "SUM", > "parameter": { > "type": "column", > "value": "PV", > "next_parameter": null > }, > "returntype": "bigint" > }, > "dependent_measure_ref": null > }, > { > "name": "USER_ID", > "function": { > "expression": "COUNT_DISTINCT", > "parameter": { > "type": "column", > "value": "USER_ID", > "next_parameter": null > }, > "returntype": "bitmap" > }, > "dependent_measure_ref": null > } > ], > "rowkey": { > "rowkey_columns": [ > { > "column": "DT", > "encoding": "dict" > }, > { > "column": "SPEC_ID", > "encoding": "dict" > }, > { > "column": "SITE_ID", > "encoding": "dict" > }, > { > "column": "DEALER_ID", > "encoding": "dict" > }, > { > "column": "CITY_ID", > "encoding": "dict" > }, > { > "column": "KEY_WORD", > "encoding": "dict" > }, > { > "column": "REF_DOMAIN", > "encoding": "dict" > }, > { > "column": "SITE_CLASS", > "encoding": "dict" > }, > { > "column": "DEVICE_ID", > "encoding": "dict" > }, > { > "column": "NEW_FLAG", > "encoding": "dict" > } > ] > }, > "signature": "8O48Zjj6YujqabUmbVcg+A==", > "last_modified": 1459996931395, > "model_name": "flw_test2", > "null_string": null, > "hbase_mapping": { > "column_family": [ > { > "name": "F1", > "columns": [ > { > "qualifier": "M", > "measure_refs": [ > "_COUNT_", > "PV" > ] > } > ] > }, > { > "name": "F2", > "columns": [ > { > "qualifier": "M", > "measure_refs": [ > "USER_ID" > ] > } > ] > } > ] > }, > "aggregation_groups": [ > { > "includes": [ > "SPEC_ID", > "CITY_ID", > "DT", > "SITE_CLASS", > "SITE_ID" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [ > [ > "SITE_CLASS", > "CITY_ID", > "SPEC_ID" > ] > ] > } > }, > { > "includes": [ > "DT", > "SITE_CLASS", > "DEVICE_ID", > "REF_DOMAIN", > "SITE_ID" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [ > [ > "DEVICE_ID", > "REF_DOMAIN", > "SITE_ID" > ] > ] > } > }, > { > "includes": [ > "DT", > "NEW_FLAG", > "SPEC_ID", > "REF_DOMAIN", > "DEALER_ID" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [] > } > }, > { > "includes": [ > "DT", > "KEY_WORD", > "SITE_ID", > "DEVICE_ID", > "REF_DOMAIN" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [] > } > } > ], > "notify_list": [], > "status_need_notify": [], > "partition_date_start": 1459814400000, > "partition_date_end": 3153600000000, > "auto_merge_time_ranges": [ > 604800000, > 2419200000 > ], > "retention_range": 0, > "engine_type": 2, > "storage_type": 2, > "override_kylin_properties": {} > } -- This message was sent by Atlassian JIRA (v6.3.4#6332)