[
https://issues.apache.org/jira/browse/HUDI-261?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Elon Azoulay updated HUDI-261:
------------------------------
Description:
{{Hudi jobs started failing with }}
{{Found commits after time :20190916210221, please rollback greater commits
first}}
This occured after a "Failed to create deltacommit inflight file" exception:
{{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException:
Failed to upsert for commit time 20190916210221 at
org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at
org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353)
at
org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228)
at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123)
at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at
org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by:
org.apache.hudi.exception.HoodieIOException: Failed to create file
gs://****************/.hoodie/20190916210223.deltacommit.inflight at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391)
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371)
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359)
at
org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417)
at
org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440)
at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... 14
more Caused by: java.io.IOException:
com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException:
412 Precondition Failed { "code" : 412, "errors" : [
{ "domain" : "global", "location" : "If-Match", "locationType" : "header",
"message" : "Precondition Failed", "reason" : "conditionNotMet" }
], "message" : "Precondition Failed" } at
com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367)
at
com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238)
at java.nio.channels.Channels$1.close(Channels.java:178) at
java.io.FilterOutputStream.close(FilterOutputStream.java:159) at
com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
at
org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66)
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388)
... 19 more Caused by:
com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException:
412 Precondition Failed}}
was:
{{Hudi jobs started failing with }}
{{Found commits after time :20190916210221, please rollback greater commits
first}}
This occured after a "Failed to create deltacommit inflight file" exception:
{{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException:
Failed to upsert for commit time 20190916210221 at
org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at
org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353)
at
org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228)
at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123)
at
org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at
org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by:
org.apache.hudi.exception.HoodieIOException: Failed to create file
gs://css-data-warehouse-hudi-ingest/hudi/data/hudi_ingest_raw/order_central_updates_latest/.hoodie/20190916210223.deltacommit.inflight
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391)
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371)
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359)
at
org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417)
at
org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440)
at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... 14
more Caused by: java.io.IOException:
com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException:
412 Precondition Failed \{ "code" : 412, "errors" : [ { "domain" : "global",
"location" : "If-Match", "locationType" : "header", "message" : "Precondition
Failed", "reason" : "conditionNotMet" } ], "message" : "Precondition Failed" }
at
com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367)
at
com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238)
at java.nio.channels.Channels$1.close(Channels.java:178) at
java.io.FilterOutputStream.close(FilterOutputStream.java:159) at
com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
at
org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66)
at
org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388)
... 19 more Caused by:
com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException:
412 Precondition Failed}}
> Failed to create deltacommit.inflight file, duplicate timestamp issue
> ---------------------------------------------------------------------
>
> Key: HUDI-261
> URL: https://issues.apache.org/jira/browse/HUDI-261
> Project: Apache Hudi (incubating)
> Issue Type: Bug
> Components: deltastreamer
> Reporter: Elon Azoulay
> Priority: Major
>
> {{Hudi jobs started failing with }}
> {{Found commits after time :20190916210221, please rollback greater commits
> first}}
>
> This occured after a "Failed to create deltacommit inflight file" exception:
> {{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException:
> Failed to upsert for commit time 20190916210221 at
> org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353)
> at
> org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228)
> at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123)
> at
> org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> at
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894)
> at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198)
> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at
> org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at
> org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by:
> org.apache.hudi.exception.HoodieIOException: Failed to create file
> gs://****************/.hoodie/20190916210223.deltacommit.inflight at
> org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391)
> at
> org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371)
> at
> org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359)
> at
> org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417)
> at
> org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440)
> at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ...
> 14 more Caused by: java.io.IOException:
> com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException:
> 412 Precondition Failed { "code" : 412, "errors" : [
> { "domain" : "global", "location" : "If-Match", "locationType" : "header",
> "message" : "Precondition Failed", "reason" : "conditionNotMet" }
> ], "message" : "Precondition Failed" } at
> com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367)
> at
> com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238)
> at java.nio.channels.Channels$1.close(Channels.java:178) at
> java.io.FilterOutputStream.close(FilterOutputStream.java:159) at
> com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127)
> at
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
> at
> org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
> at
> org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at
> org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66)
> at
> org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388)
> ... 19 more Caused by:
> com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException:
> 412 Precondition Failed}}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
