[
https://issues.apache.org/jira/browse/KYLIN-2634?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
flashput updated KYLIN-2634:
----------------------------
Description:
bq.
+----------+----------+----------+----------+----------+----------+
| ip | u_domain | u_page | r_domain | r_page | agent |
+----------+----------+----------+----------+----------+----------+
| 20000519 | 20000012 | 20118849 | 20000000 | 20000000 | 20000022 |
| 20000113 | 20000012 | 20118850 | 20000000 | 20000000 | 20000022 |
| 20000247 | 20000012 | 20118851 | 20000000 | 20000000 | 20000022 |
| 20000325 | 20000012 | 20118852 | 20000000 | 20000000 | 20000022 |
| 20000247 | 20000012 | 20118853 | 20000000 | 20000000 | 20000022 |
+----------+----------+----------+----------+----------+----------+
+----------+
| count(*) |
+----------+
| 25452592 |
+----------+
Model description:
{
"uuid": "c39058c4-3e9d-4c0c-a908-c8efef41cc91",
"last_modified": 1495117591531,
"version": "2.0.0",
"name": "LOG_PV",
"owner": "ADMIN",
"description": "",
"fact_table": "DEFAULT.LOG_DATA_170416",
"lookups": [],
"dimensions": [
{
"table": "LOG_DATA_170416",
"columns": [
"U_DOMAIN",
"U_PAGE",
"R_DOMAIN",
"R_PAGE",
"AGENT",
"IP"
]
}
],
"metrics": [
"LOG_DATA_170416.LOAD_TIME",
"LOG_DATA_170416.ARTICLE_CONTENT_HEIGHT"
],
"filter_condition": "",
"partition_desc": {
"partition_date_column": null,
"partition_time_column": null,
"partition_date_start": 0,
"partition_date_format": "yyyyMMdd",
"partition_time_format": "HH:mm:ss",
"partition_type": "APPEND",
"partition_condition_builder":
"org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
},
"capacity": "MEDIUM"
}
Cube description json:
{
"uuid": "b475f98a-1ec2-45ad-a2eb-90217aa83d9b",
"last_modified": 1495117617084,
"version": "2.0.0",
"name": "cc",
"model_name": "LOG_PV",
"description": "",
"null_string": null,
"dimensions": [
{
"name": "IP",
"table": "LOG_DATA_170416",
"column": "IP",
"derived": null
},
{
"name": "U_DOMAIN",
"table": "LOG_DATA_170416",
"column": "U_DOMAIN",
"derived": null
},
{
"name": "U_PAGE",
"table": "LOG_DATA_170416",
"column": "U_PAGE",
"derived": null
},
{
"name": "R_DOMAIN",
"table": "LOG_DATA_170416",
"column": "R_DOMAIN",
"derived": null
},
{
"name": "R_PAGE",
"table": "LOG_DATA_170416",
"column": "R_PAGE",
"derived": null
},
{
"name": "AGENT",
"table": "LOG_DATA_170416",
"column": "AGENT",
"derived": null
}
],
"measures": [
{
"name": "_COUNT_",
"function": {
"expression": "COUNT",
"parameter": {
"type": "constant",
"value": "1"
},
"returntype": "bigint"
}
}
],
"dictionaries": [],
"rowkey": {
"rowkey_columns": [
{
"column": "LOG_DATA_170416.IP",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.U_DOMAIN",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.U_PAGE",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.R_DOMAIN",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.R_PAGE",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.AGENT",
"encoding": "dict",
"isShardBy": false
}
]
},
"hbase_mapping": {
"column_family": [
{
"name": "F1",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"_COUNT_"
]
}
]
}
]
},
"aggregation_groups": [
{
"includes": [
"LOG_DATA_170416.IP",
"LOG_DATA_170416.U_DOMAIN",
"LOG_DATA_170416.U_PAGE",
"LOG_DATA_170416.R_DOMAIN",
"LOG_DATA_170416.R_PAGE",
"LOG_DATA_170416.AGENT"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [],
"joint_dims": []
}
}
],
"signature": "ALTH7M0wbIA5eA96sGbvCg==",
"notify_list": [],
"status_need_notify": [
"ERROR",
"DISCARDED",
"SUCCEED"
],
"partition_date_start": 0,
"partition_date_end": 3153600000000,
"auto_merge_time_ranges": [
604800000,
2419200000
],
"retention_range": 0,
"engine_type": 4,
"storage_type": 2,
"override_kylin_properties": {}
}
File not found Exception occured when doing Cube building .
build stops at #4 Step Name: Build Dimension Dictionary
Job error log:
java.io.FileNotFoundException: File does not exist:
/kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
at
org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
at
org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1280)
at
org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1265)
at
org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1253)
at
org.apache.hadoop.hdfs.DFSInputStream.fetchLocatedBlocksAndGetLastBlockLength(DFSInputStream.java:305)
at
org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:271)
at org.apache.hadoop.hdfs.DFSInputStream.<init>(DFSInputStream.java:263)
at org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1584)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:309)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:305)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:305)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:780)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.getInputStream(HBaseResourceStore.java:207)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.access$000(HBaseResourceStore.java:58)
at
org.apache.kylin.storage.hbase.HBaseResourceStore$2.visit(HBaseResourceStore.java:173)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:143)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.getAllResourcesImpl(HBaseResourceStore.java:168)
at
org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:182)
at
org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:175)
at
org.apache.kylin.dict.DictionaryManager.checkDupByInfo(DictionaryManager.java:374)
at
org.apache.kylin.dict.DictionaryManager.saveDictionary(DictionaryManager.java:318)
at
org.apache.kylin.cube.CubeManager.saveDictionary(CubeManager.java:234)
at
org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:68)
at
org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:54)
at
org.apache.kylin.engine.mr.steps.CreateDictionaryJob.run(CreateDictionaryJob.java:66)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
at
org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:63)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
at
org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:64)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
at
org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:142)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by:
org.apache.hadoop.ipc.RemoteException(java.io.FileNotFoundException): File does
not exist:
/kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
at
org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
at org.apache.hadoop.ipc.Client.call(Client.java:1471)
at org.apache.hadoop.ipc.Client.call(Client.java:1408)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230)
at com.sun.proxy.$Proxy30.getBlockLocations(Unknown Source)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:256)
at sun.reflect.GeneratedMethodAccessor68.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)
at com.sun.proxy.$Proxy31.getBlockLocations(Unknown Source)
at
org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1278)
... 34 more
result code:2
server configuration file added as attachment.
was:
Model description:
{
"uuid": "c39058c4-3e9d-4c0c-a908-c8efef41cc91",
"last_modified": 1495117591531,
"version": "2.0.0",
"name": "LOG_PV",
"owner": "ADMIN",
"description": "",
"fact_table": "DEFAULT.LOG_DATA_170416",
"lookups": [],
"dimensions": [
{
"table": "LOG_DATA_170416",
"columns": [
"U_DOMAIN",
"U_PAGE",
"R_DOMAIN",
"R_PAGE",
"AGENT",
"IP"
]
}
],
"metrics": [
"LOG_DATA_170416.LOAD_TIME",
"LOG_DATA_170416.ARTICLE_CONTENT_HEIGHT"
],
"filter_condition": "",
"partition_desc": {
"partition_date_column": null,
"partition_time_column": null,
"partition_date_start": 0,
"partition_date_format": "yyyyMMdd",
"partition_time_format": "HH:mm:ss",
"partition_type": "APPEND",
"partition_condition_builder":
"org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
},
"capacity": "MEDIUM"
}
Cube description json:
{
"uuid": "b475f98a-1ec2-45ad-a2eb-90217aa83d9b",
"last_modified": 1495117617084,
"version": "2.0.0",
"name": "cc",
"model_name": "LOG_PV",
"description": "",
"null_string": null,
"dimensions": [
{
"name": "IP",
"table": "LOG_DATA_170416",
"column": "IP",
"derived": null
},
{
"name": "U_DOMAIN",
"table": "LOG_DATA_170416",
"column": "U_DOMAIN",
"derived": null
},
{
"name": "U_PAGE",
"table": "LOG_DATA_170416",
"column": "U_PAGE",
"derived": null
},
{
"name": "R_DOMAIN",
"table": "LOG_DATA_170416",
"column": "R_DOMAIN",
"derived": null
},
{
"name": "R_PAGE",
"table": "LOG_DATA_170416",
"column": "R_PAGE",
"derived": null
},
{
"name": "AGENT",
"table": "LOG_DATA_170416",
"column": "AGENT",
"derived": null
}
],
"measures": [
{
"name": "_COUNT_",
"function": {
"expression": "COUNT",
"parameter": {
"type": "constant",
"value": "1"
},
"returntype": "bigint"
}
}
],
"dictionaries": [],
"rowkey": {
"rowkey_columns": [
{
"column": "LOG_DATA_170416.IP",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.U_DOMAIN",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.U_PAGE",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.R_DOMAIN",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.R_PAGE",
"encoding": "dict",
"isShardBy": false
},
{
"column": "LOG_DATA_170416.AGENT",
"encoding": "dict",
"isShardBy": false
}
]
},
"hbase_mapping": {
"column_family": [
{
"name": "F1",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"_COUNT_"
]
}
]
}
]
},
"aggregation_groups": [
{
"includes": [
"LOG_DATA_170416.IP",
"LOG_DATA_170416.U_DOMAIN",
"LOG_DATA_170416.U_PAGE",
"LOG_DATA_170416.R_DOMAIN",
"LOG_DATA_170416.R_PAGE",
"LOG_DATA_170416.AGENT"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [],
"joint_dims": []
}
}
],
"signature": "ALTH7M0wbIA5eA96sGbvCg==",
"notify_list": [],
"status_need_notify": [
"ERROR",
"DISCARDED",
"SUCCEED"
],
"partition_date_start": 0,
"partition_date_end": 3153600000000,
"auto_merge_time_ranges": [
604800000,
2419200000
],
"retention_range": 0,
"engine_type": 4,
"storage_type": 2,
"override_kylin_properties": {}
}
File not found Exception occured when doing Cube building .
build stops at #4 Step Name: Build Dimension Dictionary
Job error log:
java.io.FileNotFoundException: File does not exist:
/kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
at
org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
at
org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1280)
at
org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1265)
at
org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1253)
at
org.apache.hadoop.hdfs.DFSInputStream.fetchLocatedBlocksAndGetLastBlockLength(DFSInputStream.java:305)
at
org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:271)
at org.apache.hadoop.hdfs.DFSInputStream.<init>(DFSInputStream.java:263)
at org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1584)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:309)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:305)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:305)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:780)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.getInputStream(HBaseResourceStore.java:207)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.access$000(HBaseResourceStore.java:58)
at
org.apache.kylin.storage.hbase.HBaseResourceStore$2.visit(HBaseResourceStore.java:173)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:143)
at
org.apache.kylin.storage.hbase.HBaseResourceStore.getAllResourcesImpl(HBaseResourceStore.java:168)
at
org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:182)
at
org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:175)
at
org.apache.kylin.dict.DictionaryManager.checkDupByInfo(DictionaryManager.java:374)
at
org.apache.kylin.dict.DictionaryManager.saveDictionary(DictionaryManager.java:318)
at
org.apache.kylin.cube.CubeManager.saveDictionary(CubeManager.java:234)
at
org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:68)
at
org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:54)
at
org.apache.kylin.engine.mr.steps.CreateDictionaryJob.run(CreateDictionaryJob.java:66)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
at
org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:63)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
at
org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:64)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
at
org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:142)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by:
org.apache.hadoop.ipc.RemoteException(java.io.FileNotFoundException): File does
not exist:
/kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
at
org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
at
org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
at org.apache.hadoop.ipc.Client.call(Client.java:1471)
at org.apache.hadoop.ipc.Client.call(Client.java:1408)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230)
at com.sun.proxy.$Proxy30.getBlockLocations(Unknown Source)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:256)
at sun.reflect.GeneratedMethodAccessor68.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)
at com.sun.proxy.$Proxy31.getBlockLocations(Unknown Source)
at
org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1278)
... 34 more
result code:2
server configuration file added as attachment.
> kylin build stops while building dimension dictionary with file not found
> exception
> -----------------------------------------------------------------------------------
>
> Key: KYLIN-2634
> URL: https://issues.apache.org/jira/browse/KYLIN-2634
> Project: Kylin
> Issue Type: Bug
> Components: Job Engine
> Affects Versions: v2.0.0
> Environment: CentOS release 6.8 (Final) x64
> CDH 5.9
> Reporter: flashput
> Assignee: Dong Li
> Attachments: kylin_hive_conf.xml, kylin_job_conf.xml, kylin.log,
> kylin.properties
>
>
> bq.
> +----------+----------+----------+----------+----------+----------+
> | ip | u_domain | u_page | r_domain | r_page | agent |
> +----------+----------+----------+----------+----------+----------+
> | 20000519 | 20000012 | 20118849 | 20000000 | 20000000 | 20000022 |
> | 20000113 | 20000012 | 20118850 | 20000000 | 20000000 | 20000022 |
> | 20000247 | 20000012 | 20118851 | 20000000 | 20000000 | 20000022 |
> | 20000325 | 20000012 | 20118852 | 20000000 | 20000000 | 20000022 |
> | 20000247 | 20000012 | 20118853 | 20000000 | 20000000 | 20000022 |
> +----------+----------+----------+----------+----------+----------+
> +----------+
> | count(*) |
> +----------+
> | 25452592 |
> +----------+
> Model description:
> {
> "uuid": "c39058c4-3e9d-4c0c-a908-c8efef41cc91",
> "last_modified": 1495117591531,
> "version": "2.0.0",
> "name": "LOG_PV",
> "owner": "ADMIN",
> "description": "",
> "fact_table": "DEFAULT.LOG_DATA_170416",
> "lookups": [],
> "dimensions": [
> {
> "table": "LOG_DATA_170416",
> "columns": [
> "U_DOMAIN",
> "U_PAGE",
> "R_DOMAIN",
> "R_PAGE",
> "AGENT",
> "IP"
> ]
> }
> ],
> "metrics": [
> "LOG_DATA_170416.LOAD_TIME",
> "LOG_DATA_170416.ARTICLE_CONTENT_HEIGHT"
> ],
> "filter_condition": "",
> "partition_desc": {
> "partition_date_column": null,
> "partition_time_column": null,
> "partition_date_start": 0,
> "partition_date_format": "yyyyMMdd",
> "partition_time_format": "HH:mm:ss",
> "partition_type": "APPEND",
> "partition_condition_builder":
> "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
> },
> "capacity": "MEDIUM"
> }
> Cube description json:
> {
> "uuid": "b475f98a-1ec2-45ad-a2eb-90217aa83d9b",
> "last_modified": 1495117617084,
> "version": "2.0.0",
> "name": "cc",
> "model_name": "LOG_PV",
> "description": "",
> "null_string": null,
> "dimensions": [
> {
> "name": "IP",
> "table": "LOG_DATA_170416",
> "column": "IP",
> "derived": null
> },
> {
> "name": "U_DOMAIN",
> "table": "LOG_DATA_170416",
> "column": "U_DOMAIN",
> "derived": null
> },
> {
> "name": "U_PAGE",
> "table": "LOG_DATA_170416",
> "column": "U_PAGE",
> "derived": null
> },
> {
> "name": "R_DOMAIN",
> "table": "LOG_DATA_170416",
> "column": "R_DOMAIN",
> "derived": null
> },
> {
> "name": "R_PAGE",
> "table": "LOG_DATA_170416",
> "column": "R_PAGE",
> "derived": null
> },
> {
> "name": "AGENT",
> "table": "LOG_DATA_170416",
> "column": "AGENT",
> "derived": null
> }
> ],
> "measures": [
> {
> "name": "_COUNT_",
> "function": {
> "expression": "COUNT",
> "parameter": {
> "type": "constant",
> "value": "1"
> },
> "returntype": "bigint"
> }
> }
> ],
> "dictionaries": [],
> "rowkey": {
> "rowkey_columns": [
> {
> "column": "LOG_DATA_170416.IP",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.U_DOMAIN",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.U_PAGE",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.R_DOMAIN",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.R_PAGE",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.AGENT",
> "encoding": "dict",
> "isShardBy": false
> }
> ]
> },
> "hbase_mapping": {
> "column_family": [
> {
> "name": "F1",
> "columns": [
> {
> "qualifier": "M",
> "measure_refs": [
> "_COUNT_"
> ]
> }
> ]
> }
> ]
> },
> "aggregation_groups": [
> {
> "includes": [
> "LOG_DATA_170416.IP",
> "LOG_DATA_170416.U_DOMAIN",
> "LOG_DATA_170416.U_PAGE",
> "LOG_DATA_170416.R_DOMAIN",
> "LOG_DATA_170416.R_PAGE",
> "LOG_DATA_170416.AGENT"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [],
> "joint_dims": []
> }
> }
> ],
> "signature": "ALTH7M0wbIA5eA96sGbvCg==",
> "notify_list": [],
> "status_need_notify": [
> "ERROR",
> "DISCARDED",
> "SUCCEED"
> ],
> "partition_date_start": 0,
> "partition_date_end": 3153600000000,
> "auto_merge_time_ranges": [
> 604800000,
> 2419200000
> ],
> "retention_range": 0,
> "engine_type": 4,
> "storage_type": 2,
> "override_kylin_properties": {}
> }
> File not found Exception occured when doing Cube building .
> build stops at #4 Step Name: Build Dimension Dictionary
> Job error log:
> java.io.FileNotFoundException: File does not exist:
> /kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
> at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
> at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
> at
> org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
> at
> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
> at
> org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
> at
> org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1280)
> at
> org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1265)
> at
> org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1253)
> at
> org.apache.hadoop.hdfs.DFSInputStream.fetchLocatedBlocksAndGetLastBlockLength(DFSInputStream.java:305)
> at
> org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:271)
> at org.apache.hadoop.hdfs.DFSInputStream.<init>(DFSInputStream.java:263)
> at org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1584)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:309)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:305)
> at
> org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:305)
> at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:780)
> at
> org.apache.kylin.storage.hbase.HBaseResourceStore.getInputStream(HBaseResourceStore.java:207)
> at
> org.apache.kylin.storage.hbase.HBaseResourceStore.access$000(HBaseResourceStore.java:58)
> at
> org.apache.kylin.storage.hbase.HBaseResourceStore$2.visit(HBaseResourceStore.java:173)
> at
> org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:143)
> at
> org.apache.kylin.storage.hbase.HBaseResourceStore.getAllResourcesImpl(HBaseResourceStore.java:168)
> at
> org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:182)
> at
> org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:175)
> at
> org.apache.kylin.dict.DictionaryManager.checkDupByInfo(DictionaryManager.java:374)
> at
> org.apache.kylin.dict.DictionaryManager.saveDictionary(DictionaryManager.java:318)
> at
> org.apache.kylin.cube.CubeManager.saveDictionary(CubeManager.java:234)
> at
> org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:68)
> at
> org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:54)
> at
> org.apache.kylin.engine.mr.steps.CreateDictionaryJob.run(CreateDictionaryJob.java:66)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
> at
> org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:63)
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
> at
> org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:64)
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
> at
> org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:142)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(java.io.FileNotFoundException): File
> does not exist:
> /kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
> at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
> at
> org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
> at
> org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
> at org.apache.hadoop.ipc.Client.call(Client.java:1471)
> at org.apache.hadoop.ipc.Client.call(Client.java:1408)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230)
> at com.sun.proxy.$Proxy30.getBlockLocations(Unknown Source)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:256)
> at sun.reflect.GeneratedMethodAccessor68.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)
> at com.sun.proxy.$Proxy31.getBlockLocations(Unknown Source)
> at
> org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1278)
> ... 34 more
> result code:2
> server configuration file added as attachment.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
