[GitHub] [airflow] pankajastro commented on a diff in pull request #29835: Provider Google Cloud - Add bigquery bi engine reservation operators
pankajastro commented on code in PR #29835: URL: https://github.com/apache/airflow/pull/29835#discussion_r1127687575 ## airflow/providers/google/cloud/hooks/bigquery_reservation.py: ## @@ -0,0 +1,123 @@ +# Review Comment: +1 for `hooks/bigquery_biengine.py` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] pankajastro commented on a diff in pull request #29835: Provider Google Cloud - Add bigquery bi engine reservation operators
pankajastro commented on code in PR #29835: URL: https://github.com/apache/airflow/pull/29835#discussion_r112636 ## airflow/providers/google/cloud/hooks/bigquery_reservation.py: ## @@ -0,0 +1,125 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a BigQuery Reservation Hook.""" +from __future__ import annotations + +from typing import Sequence + +from google.cloud.bigquery_reservation_v1 import ReservationServiceClient + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class BigQueryReservationServiceHook(GoogleBaseHook): +""" +Hook for Google Bigquery Reservation API. + +:param gcp_conn_id: The Airflow connection used for GCP credentials. +:param location: The location of the BigQuery resource. +:param impersonation_chain: This is the optional service account to impersonate using short term +credentials. +""" + +conn_name_attr = "gcp_conn_id" +default_conn_name = "google_cloud_bigquery_reservation_default" Review Comment: I would prefer to use `google_cloud_default` name and not introduce a new one unless it absolutely requires. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] pankajastro commented on a diff in pull request #29835: Provider Google Cloud - Add bigquery bi engine reservation operators
pankajastro commented on code in PR #29835: URL: https://github.com/apache/airflow/pull/29835#discussion_r1126307197 ## airflow/providers/google/cloud/hooks/bigquery_reservation.py: ## @@ -0,0 +1,125 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a BigQuery Reservation Hook.""" +from __future__ import annotations + +from typing import Sequence + +from google.cloud.bigquery_reservation_v1 import ReservationServiceClient + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class BigQueryReservationServiceHook(GoogleBaseHook): +""" +Hook for Google Bigquery Reservation API. + +:param gcp_conn_id: The Airflow connection used for GCP credentials. +:param location: The location of the BigQuery resource. +:param impersonation_chain: This is the optional service account to impersonate using short term +credentials. +""" + +conn_name_attr = "gcp_conn_id" +default_conn_name = "google_cloud_bigquery_reservation_default" +hook_name = "Google Bigquery Reservation" + +def __init__( +self, +gcp_conn_id: str = GoogleBaseHook.default_conn_name, +location: str | None = None, +impersonation_chain: str | Sequence[str] | None = None, +) -> None: +super().__init__( +gcp_conn_id=gcp_conn_id, +impersonation_chain=impersonation_chain, +) +self.location = location +self._client: ReservationServiceClient | None = None + +@staticmethod +def _convert_gb_to_kb(value: int) -> int: +""" +Convert GB value to KB. + +:param value: Value to convert +""" +return value * 1073741824 + +def get_client(self) -> ReservationServiceClient: +""" +Get reservation service client. + +:return: Google Bigquery Reservation client +""" +if not self._client: +self._client = ReservationServiceClient( +credentials=self.get_credentials(), client_info=CLIENT_INFO +) +return self._client + +@GoogleBaseHook.fallback_to_default_project_id +def create_bi_reservation(self, project_id: str, size: int) -> None: Review Comment: I see - then I'm ok to keep it in its current form. thanks! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] pankajastro commented on a diff in pull request #29835: Provider Google Cloud - Add bigquery bi engine reservation operators
pankajastro commented on code in PR #29835: URL: https://github.com/apache/airflow/pull/29835#discussion_r1123551136 ## airflow/providers/google/cloud/hooks/bigquery_reservation.py: ## @@ -0,0 +1,125 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a BigQuery Reservation Hook.""" +from __future__ import annotations + +from typing import Sequence + +from google.cloud.bigquery_reservation_v1 import ReservationServiceClient + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class BigQueryReservationServiceHook(GoogleBaseHook): +""" +Hook for Google Bigquery Reservation API. + +:param gcp_conn_id: The Airflow connection used for GCP credentials. +:param location: The location of the BigQuery resource. +:param impersonation_chain: This is the optional service account to impersonate using short term +credentials. +""" + +conn_name_attr = "gcp_conn_id" +default_conn_name = "google_cloud_bigquery_reservation_default" +hook_name = "Google Bigquery Reservation" + +def __init__( +self, +gcp_conn_id: str = GoogleBaseHook.default_conn_name, +location: str | None = None, +impersonation_chain: str | Sequence[str] | None = None, +) -> None: +super().__init__( +gcp_conn_id=gcp_conn_id, +impersonation_chain=impersonation_chain, +) +self.location = location +self._client: ReservationServiceClient | None = None + +@staticmethod +def _convert_gb_to_kb(value: int) -> int: +""" +Convert GB value to KB. + +:param value: Value to convert +""" +return value * 1073741824 + +def get_client(self) -> ReservationServiceClient: +""" +Get reservation service client. + +:return: Google Bigquery Reservation client +""" +if not self._client: +self._client = ReservationServiceClient( +credentials=self.get_credentials(), client_info=CLIENT_INFO +) +return self._client + +@GoogleBaseHook.fallback_to_default_project_id +def create_bi_reservation(self, project_id: str, size: int) -> None: Review Comment: hmm, in that case I would prefer to have this in `BigQueryReservationServiceHook` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] pankajastro commented on a diff in pull request #29835: Provider Google Cloud - Add bigquery bi engine reservation operators
pankajastro commented on code in PR #29835: URL: https://github.com/apache/airflow/pull/29835#discussion_r1123530878 ## airflow/providers/google/cloud/hooks/bigquery_reservation.py: ## @@ -0,0 +1,125 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a BigQuery Reservation Hook.""" +from __future__ import annotations + +from typing import Sequence + +from google.cloud.bigquery_reservation_v1 import ReservationServiceClient + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class BigQueryReservationServiceHook(GoogleBaseHook): +""" +Hook for Google Bigquery Reservation API. + +:param gcp_conn_id: The Airflow connection used for GCP credentials. +:param location: The location of the BigQuery resource. +:param impersonation_chain: This is the optional service account to impersonate using short term +credentials. +""" + +conn_name_attr = "gcp_conn_id" +default_conn_name = "google_cloud_bigquery_reservation_default" +hook_name = "Google Bigquery Reservation" + +def __init__( +self, +gcp_conn_id: str = GoogleBaseHook.default_conn_name, +location: str | None = None, +impersonation_chain: str | Sequence[str] | None = None, +) -> None: +super().__init__( +gcp_conn_id=gcp_conn_id, +impersonation_chain=impersonation_chain, +) +self.location = location +self._client: ReservationServiceClient | None = None + +@staticmethod +def _convert_gb_to_kb(value: int) -> int: +""" +Convert GB value to KB. + +:param value: Value to convert +""" +return value * 1073741824 + +def get_client(self) -> ReservationServiceClient: +""" +Get reservation service client. + +:return: Google Bigquery Reservation client +""" +if not self._client: +self._client = ReservationServiceClient( +credentials=self.get_credentials(), client_info=CLIENT_INFO +) +return self._client + +@GoogleBaseHook.fallback_to_default_project_id +def create_bi_reservation(self, project_id: str, size: int) -> None: Review Comment: do we need project_id args here? can we access this with `self.project_id` since this is a base class property? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] pankajastro commented on a diff in pull request #29835: Provider Google Cloud - Add bigquery bi engine reservation operators
pankajastro commented on code in PR #29835: URL: https://github.com/apache/airflow/pull/29835#discussion_r1122802132 ## airflow/providers/google/cloud/hooks/bigquery_reservation.py: ## @@ -0,0 +1,129 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains a BigQuery Reservation Hook.""" +from __future__ import annotations + +from typing import Sequence + +from google.cloud.bigquery_reservation_v1 import ReservationServiceClient + +from airflow.exceptions import AirflowException +from airflow.providers.google.common.consts import CLIENT_INFO +from airflow.providers.google.common.hooks.base_google import GoogleBaseHook + + +class BigQueryReservationServiceHook(GoogleBaseHook): +""" +Hook for Google Bigquery Reservation API. + +:param gcp_conn_id: The Airflow connection used for GCP credentials. +:param location: The location of the BigQuery resource. +:param delegate_to: This performs a task on one host with reference to other hosts. +:param impersonation_chain: This is the optional service account to impersonate using short term +credentials. +""" + +conn_name_attr = "gcp_conn_id" +default_conn_name = "google_cloud_bigquery_reservation_default" +conn_type = "gcpbigqueryreservation" +hook_name = "Google Bigquery Reservation" + +def __init__( +self, +gcp_conn_id: str = GoogleBaseHook.default_conn_name, +location: str | None = None, +delegate_to: str | None = None, Review Comment: in `GoogleBaseHook` we have deprecated `delegate_to` so can we remove this from here? ## airflow/providers/google/cloud/operators/bigquery_reservation.py: ## @@ -0,0 +1,150 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module contains Google BigQuery Reservation Service operators.""" +from __future__ import annotations + +from typing import Any, Sequence + +from airflow.models import BaseOperator +from airflow.providers.google.cloud.hooks.bigquery_reservation import BigQueryReservationServiceHook + + +class BigQueryBiEngineReservationCreateOperator(BaseOperator): +""" +Create or Update BI engine reservation. + +:param location: The BI engine reservation location. (templated) +:param size: The BI Engine reservation memory size (GB). (templated) +:param project_id: (Optional) The name of the project where the reservation +will be attached from. (templated) +:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud. (templated) +:param delegate_to: Account to impersonate using domain-wide delegation of authority, Review Comment: since we have deprecated `delegate_to` in google provider so can avoid adding it here? ## airflow/providers/google/cloud/example_dags/example_bigquery_reservation_bi_engine.py: ## @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distribu
