[ https://issues.apache.org/jira/browse/IMPALA-9058?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16953247#comment-16953247 ]
Sahil Takiar commented on IMPALA-9058: -------------------------------------- The full stack-trace from the HMS logs is: {code:java} 2019-10-15T22:48:20,319 ERROR [HMSHandler #3] utils.MetaStoreUtils: Got exception: java.io.FileNotFoundException getVersionMarkerItem on ../VERSION: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 8T9IS939MDI7ASOB0IJCC34J3NVV4KQNSO5AEMVJF66Q9ASUAAJG) java.io.FileNotFoundException: getVersionMarkerItem on ../VERSION: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 8T9IS939MDI7ASOB0IJCC34J3NVV4KQNSO5AEMVJF66Q9ASUAAJG) at org.apache.hadoop.fs.s3a.S3AUtils.translateDynamoDBException(S3AUtils.java:424) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:206) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:314) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:406) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:310) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:285) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.queryVersionMarker(DynamoDBMetadataStore.java:1873) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.getVersionMarkerItem(DynamoDBMetadataStore.java:1830) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.initTable(DynamoDBMetadataStore.java:1773) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.initialize(DynamoDBMetadataStore.java:439) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.s3guard.S3Guard.getMetadataStore(S3Guard.java:109) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:399) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3315) ~[hadoop-common-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:136) ~[hadoop-common-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3364) ~[hadoop-common-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3332) ~[hadoop-common-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:491) ~[hadoop-common-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361) ~[hadoop-common-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.hive.metastore.Warehouse.getFs(Warehouse.java:116) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at org.apache.hadoop.hive.metastore.Warehouse.getDnsPath(Warehouse.java:142) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at org.apache.hadoop.hive.metastore.Warehouse.getDnsPath(Warehouse.java:148) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createLocationForAddedPartition(HiveMetaStore.java:4276) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.access$500(HiveMetaStore.java:272) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler$13$1.run(HiveMetaStore.java:3920) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at java.security.AccessController.doPrivileged(Native Method) [?:1.8.0_144] at javax.security.auth.Subject.doAs(Subject.java:422) [?:1.8.0_144] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730) [hadoop-common-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler$13.call(HiveMetaStore.java:3916) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler$13.call(HiveMetaStore.java:3913) [hive-exec-3.1.0.7.1.0.0-33.jar:3.1.0.7.1.0.0-33] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_144] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_144] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_144] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_144] Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 8T9IS939MDI7ASOB0IJCC34J3NVV4KQNSO5AEMVJF66Q9ASUAAJG) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4279) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4246) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeGetItem(AmazonDynamoDBClient.java:2054) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.getItem(AmazonDynamoDBClient.java:2020) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.document.internal.GetItemImpl.doLoadItem(GetItemImpl.java:77) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.document.internal.GetItemImpl.getItemOutcome(GetItemImpl.java:46) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.document.internal.GetItemImpl.getItem(GetItemImpl.java:88) ~[aws-java-sdk-bundle-1.11.563.jar:?] at com.amazonaws.services.dynamodbv2.document.Table.getItem(Table.java:597) ~[aws-java-sdk-bundle-1.11.563.jar:?] at org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.lambda$queryVersionMarker$11(DynamoDBMetadataStore.java:1875) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109) ~[hadoop-aws-3.1.1.7.1.0.0-33.jar:?] ... 31 more {code} > S3 tests failing with FileNotFoundException getVersionMarkerItem on ../VERSION > ------------------------------------------------------------------------------ > > Key: IMPALA-9058 > URL: https://issues.apache.org/jira/browse/IMPALA-9058 > Project: IMPALA > Issue Type: Test > Reporter: Sahil Takiar > Assignee: Sahil Takiar > Priority: Major > > I've seen this happen several times now, S3 tests intermittently fail with an > error such as: > {code:java} > Query aborted:InternalException: Error adding partitions E CAUSED BY: > MetaException: java.io.IOException: Got exception: > java.io.FileNotFoundException getVersionMarkerItem on ../VERSION: > com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested > resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: > ResourceNotFoundException; Request ID: > 8T9IS939MDI7ASOB0IJCC34J3NVV4KQNSO5AEMVJF66Q9ASUAAJG) {code} -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-all-unsubscr...@impala.apache.org For additional commands, e-mail: issues-all-h...@impala.apache.org