This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 51e854bde feat(csharp/src/Drivers/BigQuery): Add support for
specifying a location (#3494)
51e854bde is described below
commit 51e854bde9299863def0ef5b0dba2541d0cfa431
Author: davidhcoe <[email protected]>
AuthorDate: Mon Sep 29 13:12:08 2025 -0400
feat(csharp/src/Drivers/BigQuery): Add support for specifying a location
(#3494)
In the current iteration, no DefaultLocation is specified for the
internal BigQueryClient. Since this is not specified, the BigQuery API
makes this default to "US" which is causing failures for some users.
This PR adds a new parameter to set the DefaultLocation for the
BigQueryClient, which then uses that same value for creating datasets
and other location-based activitities.
---------
Co-authored-by: David Coe <>
---
csharp/src/Drivers/BigQuery/BigQueryConnection.cs | 28 ++++++++++
csharp/src/Drivers/BigQuery/BigQueryParameters.cs | 61 +++++++++++++++++++++-
csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 8 ++-
.../Drivers/BigQuery/BigQueryTestConfiguration.cs | 3 ++
.../test/Drivers/BigQuery/BigQueryTestingUtils.cs | 5 ++
5 files changed, 102 insertions(+), 3 deletions(-)
diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
index ffb562d78..43082d8be 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
@@ -90,6 +90,13 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
{
RetryDelayMs = delay;
}
+
+ if
(this.properties.TryGetValue(BigQueryParameters.DefaultClientLocation, out
string? location) &&
+ !string.IsNullOrEmpty(location) &&
+ BigQueryConstants.ValidLocations.Any(l => l.Equals(location,
StringComparison.OrdinalIgnoreCase)))
+ {
+ DefaultClientLocation = location;
+ }
}
private bool TryInitTracerProvider(out FileActivityListener?
fileActivityListener)
@@ -132,6 +139,9 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
internal int RetryDelayMs { get; private set; } = 200;
+ // if this value is null, the BigQuery API chooses the location
(typically the `US` multi-region)
+ internal string? DefaultClientLocation { get; private set; }
+
public override string AssemblyVersion =>
BigQueryUtils.BigQueryAssemblyVersion;
public override string AssemblyName =>
BigQueryUtils.BigQueryAssemblyName;
@@ -203,6 +213,24 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
GoogleCredential = Credential
};
+ if (!string.IsNullOrEmpty(DefaultClientLocation))
+ {
+ // If the user selects a public dataset (from a
multi-region) but sets this
+ // value to a specific location like us-east4, then there
is an error produced
+ // that the caller doesn't have permission to call to the
public dataset.
+ // Example:
+ // Access Denied: Table
bigquery-public-data:blockchain_analytics_ethereum_mainnet_us.accounts:
+ // User does not have permission to query table
bigquery-public-data:blockchain_analytics_ethereum_mainnet_us.accounts,
+ // or perhaps it does not exist.'
+
+ bigQueryClientBuilder.DefaultLocation =
DefaultClientLocation;
+
activity?.AddBigQueryParameterTag(BigQueryParameters.DefaultClientLocation,
DefaultClientLocation);
+ }
+ else
+ {
+ activity?.AddBigQueryTag("client.default_location", null);
+ }
+
BigQueryClient client = bigQueryClientBuilder.Build();
if (clientTimeout.HasValue)
diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
index a02173e0d..ec18e55f1 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
@@ -40,6 +40,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
public const string JsonCredential =
"adbc.bigquery.auth_json_credential";
public const string LargeDecimalsAsString =
"adbc.bigquery.large_decimals_as_string";
public const string LargeResultsDataset =
"adbc.bigquery.large_results_dataset";
+ public const string DefaultClientLocation =
"adbc.bigquery.default_client_location";
public const string LargeResultsDestinationTable =
"adbc.bigquery.large_results_destination_table";
public const string MaxFetchConcurrency =
"adbc.bigquery.max_fetch_concurrency";
public const string MaximumRetryAttempts =
"adbc.bigquery.maximum_retries";
@@ -54,7 +55,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
// these values are safe to log any time
private static HashSet<string> safeToLog = new
HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
- AllowLargeResults, AuthenticationType, BillingProjectId, ClientId,
ClientTimeout,
+ AllowLargeResults, AuthenticationType, BillingProjectId, ClientId,
ClientTimeout, DefaultClientLocation, EvaluationKind,
GetQueryResultsOptionsTimeout,
EvaluationKind, GetQueryResultsOptionsTimeout,
IncludeConstraintsWithGetObjects,
IncludePublicProjectId, LargeDecimalsAsString,
LargeResultsDataset, LargeResultsDestinationTable,
MaxFetchConcurrency, MaximumRetryAttempts, ProjectId,
RetryDelayMs, StatementIndex,
@@ -95,5 +96,63 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
public const string DefaultLargeDatasetId = "_bqadbc_temp_tables";
public const string PublicProjectId = "bigquery-public-data";
+
+ // this is what the BigQuery API uses as the default location
+ public const string DefaultClientLocation = "US";
+
+ // from
https://cloud.google.com/bigquery/docs/locations#locations_and_regions as of
Sept 28, 2025
+ public static IReadOnlyList<string> ValidLocations = new List<string>()
+ {
+ //multi-regions
+ "US",
+ "EU",
+ // Americas
+ "us-east5",
+ "us-south1",
+ "us-central1",
+ "us-west4",
+ "us-west2",
+ "northamerica-south1",
+ "northamerica-northeast1",
+ "us-east4",
+ "us-west1",
+ "us-west3",
+ "southamerica-east1",
+ "southamerica-west1",
+ "us-east1",
+ "northamerica-northeast2",
+ // Asia Pacific
+ "asia-south2",
+ "asia-east2",
+ "asia-southeast2",
+ "australia-southeast2",
+ "asia-south1",
+ "asia-northeast2",
+ "asia-northeast3",
+ "asia-southeast1",
+ "australia-southeast1",
+ "asia-east1",
+ "asia-northeast1",
+ // Europe
+ "europe-west1",
+ "europe-west10",
+ "europe-north1",
+ "europe-west3",
+ "europe-west2",
+ "europe-southwest1",
+ "europe-west8",
+ "europe-west4",
+ "europe-west9",
+ "europe-north2",
+ "europe-west12",
+ "europe-central2",
+ "europe-west6",
+ // Middle East
+ "me-central2",
+ "me-central1",
+ "me-west1",
+ // Africa
+ "africa-south1"
+ };
}
}
diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
index eabdcc4da..880d8b79f 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
@@ -94,7 +94,6 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
activity?.AddConditionalTag(SemanticConventions.Db.Query.Text,
SqlQuery, this.bigQueryConnection.IsSafeToTrace);
BigQueryJob job = await Client.CreateQueryJobAsync(SqlQuery,
null, queryOptions);
-
JobReference jobReference = job.Reference;
GetQueryResultsOptions getQueryResultsOptions = new
GetQueryResultsOptions();
@@ -484,8 +483,13 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
try
{
activity?.AddBigQueryTag("large_results.dataset.try_create", datasetId);
+
activity?.AddBigQueryTag("large_results.dataset.try_create_region",
this.Client.DefaultLocation);
DatasetReference reference =
this.Client.GetDatasetReference(datasetId);
+ // The location is not set here because it will use the
DefaultLocation from the client.
+ // Similar to the DefaultLocation for the client, if the
caller attempts to use a public
+ // dataset from a multi-region but set the destination to
a specific location,
+ // a similar permission error is thrown.
BigQueryDataset bigQueryDataset = new
BigQueryDataset(this.Client, new Dataset()
{
DatasetReference = reference,
@@ -498,7 +502,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
catch (Exception ex)
{
activity?.AddException(ex);
- throw new AdbcException($"Could not create dataset
{datasetId}", ex);
+ throw new AdbcException($"Could not create dataset
{datasetId} in {this.Client.DefaultLocation}", ex);
}
}
diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
index b88a8b7f2..74f856e10 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
@@ -120,6 +120,9 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[JsonPropertyName("evaluationKind")]
public string EvaluationKind { get; set; } = string.Empty;
+ [JsonPropertyName("location")]
+ public string? ClientLocation { get; set; }
+
/// <summary>
/// How structs should be handled by the ADO.NET client for this
environment.
/// </summary>
diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
index e3590212e..394502895 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
@@ -141,6 +141,11 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
parameters.Add(BigQueryParameters.AllowLargeResults,
testEnvironment.AllowLargeResults.ToString());
}
+ if (!string.IsNullOrEmpty(testEnvironment.ClientLocation))
+ {
+ parameters.Add(BigQueryParameters.DefaultClientLocation,
testEnvironment.ClientLocation!);
+ }
+
parameters.Add(BigQueryParameters.IncludeConstraintsWithGetObjects,
testEnvironment.IncludeTableConstraints.ToString());
parameters.Add(BigQueryParameters.IncludePublicProjectId,
testEnvironment.IncludePublicProjectId.ToString());
