[ https://issues.apache.org/jira/browse/HUDI-261?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Vinoth Chandar updated HUDI-261: -------------------------------- Description: {{Hudi jobs started failing with }} {{Found commits after time :20190916210221, please rollback greater commits first}} This occured after a "Failed to create deltacommit inflight file" exception: {code:shell} {{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException: Failed to upsert for commit time 20190916210221 at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353) at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: org.apache.hudi.exception.HoodieIOException: Failed to create file gs://****************/.hoodie/20190916210223.deltacommit.inflight at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359) at org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417) at org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440) at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... 14 more Caused by: java.io.IOException: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed { "code" : 412, "errors" : [ { "domain" : "global", "location" : "If-Match", "locationType" : "header", "message" : "Precondition Failed", "reason" : "conditionNotMet" } ], "message" : "Precondition Failed" } at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367) at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238) at java.nio.channels.Channels$1.close(Channels.java:178) at java.io.FilterOutputStream.close(FilterOutputStream.java:159) at com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388) ... 19 more Caused by: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed}} {code} was: {{Hudi jobs started failing with }} {{Found commits after time :20190916210221, please rollback greater commits first}} This occured after a "Failed to create deltacommit inflight file" exception: {{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException: Failed to upsert for commit time 20190916210221 at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353) at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: org.apache.hudi.exception.HoodieIOException: Failed to create file gs://****************/.hoodie/20190916210223.deltacommit.inflight at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359) at org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417) at org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440) at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... 14 more Caused by: java.io.IOException: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed { "code" : 412, "errors" : [ { "domain" : "global", "location" : "If-Match", "locationType" : "header", "message" : "Precondition Failed", "reason" : "conditionNotMet" } ], "message" : "Precondition Failed" } at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367) at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238) at java.nio.channels.Channels$1.close(Channels.java:178) at java.io.FilterOutputStream.close(FilterOutputStream.java:159) at com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388) ... 19 more Caused by: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed}} > Failed to create deltacommit.inflight file, duplicate timestamp issue > --------------------------------------------------------------------- > > Key: HUDI-261 > URL: https://issues.apache.org/jira/browse/HUDI-261 > Project: Apache Hudi (incubating) > Issue Type: Bug > Components: deltastreamer > Reporter: Elon Azoulay > Priority: Major > > {{Hudi jobs started failing with }} > {{Found commits after time :20190916210221, please rollback greater commits > first}} > > This occured after a "Failed to create deltacommit inflight file" exception: > {code:shell} > {{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException: > Failed to upsert for commit time 20190916210221 at > org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at > org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353) > at > org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228) > at > org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123) > at > org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) at > org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) > at > org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) > at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) > at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at > org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at > org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: > org.apache.hudi.exception.HoodieIOException: Failed to create file > gs://****************/.hoodie/20190916210223.deltacommit.inflight at > org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391) > at > org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371) > at > org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359) > at > org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417) > at > org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440) > at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... > 14 more Caused by: java.io.IOException: > com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: > 412 Precondition Failed { "code" : 412, "errors" : [ > { "domain" : "global", "location" : "If-Match", "locationType" : "header", > "message" : "Precondition Failed", "reason" : "conditionNotMet" } > ], "message" : "Precondition Failed" } at > com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367) > at > com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238) > at java.nio.channels.Channels$1.close(Channels.java:178) at > java.io.FilterOutputStream.close(FilterOutputStream.java:159) at > com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127) > at > org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) > at > org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at > org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) > at > org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at > org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66) > at > org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388) > ... 19 more Caused by: > com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: > 412 Precondition Failed}} > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005)