This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new ed383fa4a3b Logs the BigQuery error that caused the load job failure
(resolves #1… (#36068)
ed383fa4a3b is described below
commit ed383fa4a3bdafe8be421478cdeabad103b7756d
Author: Ian Mburu <[email protected]>
AuthorDate: Mon Sep 8 16:54:53 2025 +0200
Logs the BigQuery error that caused the load job failure (resolves #1…
(#36068)
* Logs the BigQuery error that caused the load job failure (resolves #18419)
* Checks that the BigQuery root cause message to be logged is not empty
(resolves #18419)
---
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 129c8314fc8..d468ffbea43 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -25,6 +25,7 @@ import com.google.api.client.util.BackOffUtils;
import com.google.api.client.util.Sleeper;
import com.google.api.services.bigquery.model.Clustering;
import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.ErrorProto;
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.JobStatus;
@@ -205,6 +206,7 @@ public class BigQueryHelpers {
void runJob() throws IOException {
++currentAttempt;
if (!shouldRetry()) {
+ logBigQueryError(lastJobAttempted);
throw new RuntimeException(
String.format(
"Failed to create job with prefix %s, "
@@ -281,6 +283,21 @@ public class BigQueryHelpers {
boolean shouldRetry() {
return currentAttempt < maxRetries + 1;
}
+
+ void logBigQueryError(@Nullable Job job) {
+ if (job == null || !parseStatus(job).equals(Status.FAILED)) {
+ return;
+ }
+
+ List<ErrorProto> jobErrors = job.getStatus().getErrors();
+ String finalError = job.getStatus().getErrorResult().getMessage();
+ String causativeError =
+ jobErrors != null && !jobErrors.isEmpty()
+ ? String.format(" due to: %s", jobErrors.get(jobErrors.size() -
1).getMessage())
+ : "";
+
+ LOG.error(String.format("BigQuery Error : %s %s", finalError,
causativeError));
+ }
}
static class RetryJobId implements Serializable {
