Google Ads Hook: Support newer versions of the google-ads library (#17160)

infused-kim · web-flow · commit 5d2224795b35 · 2021-07-25T22:55:49.000+02:00
diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst
@@ -21,6 +21,22 @@ Changelog
 4.1.0
 .....
 
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+* ``Updated GoogleAdsHook to support newer API versions after google deprecated v5. Google Ads v8 is the new default API. (#17111)``
+
+.. warning:: The underlying google-ads library had breaking changes.
+   
+   Previously the google ads library returned data as native protobuf messages. Now it returns data as proto-plus objects that behave more like conventional Python objects. 
+
+   To preserve compatibility the hook's `search()` converts the data back to native protobuf before returning it. Your existing operators *should* work as before, but due to the urgency of the v5 API being deprecated it was not tested too thoroughly. Therefore you should carefully evaluate your operator and hook functionality with this new version.
+
+   In order to use the API's new proto-plus format, you can use the `search_proto_plus()` method.
+
+   For more information, please consult `google-ads migration document <https://developers.google.com/google-ads/api/docs/client-libs/python/library-version-10>`__: 
+
+
 Features
 ~~~~~~~~
 
diff --git a/airflow/providers/google/ads/hooks/ads.py b/airflow/providers/google/ads/hooks/ads.py
@@ -23,9 +23,9 @@
     from functools import cached_property
 except ImportError:
     from cached_property import cached_property
-from google.ads.google_ads.client import GoogleAdsClient
-from google.ads.google_ads.errors import GoogleAdsException
-from google.ads.google_ads.v2.types import GoogleAdsRow
+from google.ads.googleads.client import GoogleAdsClient
+from google.ads.googleads.errors import GoogleAdsException
+from google.ads.googleads.v8.services.types.google_ads_service import GoogleAdsRow
 from google.api_core.page_iterator import GRPCIterator
 from google.auth.exceptions import GoogleAuthError
 from googleapiclient.discovery import Resource
@@ -76,7 +76,7 @@ class GoogleAdsHook(BaseHook):
     :rtype: list[GoogleAdsRow]
     """
 
-    default_api_version = "v5"
+    default_api_version = "v8"
 
     def __init__(
         self,
@@ -90,15 +90,92 @@ def __init__(
         self.google_ads_conn_id = google_ads_conn_id
         self.google_ads_config: Dict[str, Any] = {}
 
+    def search(
+        self, client_ids: List[str], query: str, page_size: int = 10000, **kwargs
+    ) -> List[GoogleAdsRow]:
+        """
+        Pulls data from the Google Ads API and returns it as native protobuf
+        message instances (those seen in versions prior to 10.0.0 of the
+        google-ads library).
+
+        This method is for backwards compatibility with older versions of the
+        google_ads_hook.
+
+        Check out the search_proto_plus method to get API results in the new
+        default format of the google-ads library since v10.0.0 that behave
+        more like conventional python object (using proto-plus-python).
+
+        :param client_ids: Google Ads client ID(s) to query the API for.
+        :type client_ids: List[str]
+        :param query: Google Ads Query Language query.
+        :type query: str
+        :param page_size: Number of results to return per page. Max 10000.
+        :type page_size: int
+        :return: Google Ads API response, converted to Google Ads Row objects
+        :rtype: list[GoogleAdsRow]
+        """
+        data_proto_plus = self._search(client_ids, query, page_size, **kwargs)
+        data_native_pb = [row._pb for row in data_proto_plus]
+
+        return data_native_pb
+
+    def search_proto_plus(
+        self, client_ids: List[str], query: str, page_size: int = 10000, **kwargs
+    ) -> List[GoogleAdsRow]:
+        """
+        Pulls data from the Google Ads API and returns it as proto-plus-python
+        message instances that behave more like conventional python objects.
+
+        :param client_ids: Google Ads client ID(s) to query the API for.
+        :type client_ids: List[str]
+        :param query: Google Ads Query Language query.
+        :type query: str
+        :param page_size: Number of results to return per page. Max 10000.
+        :type page_size: int
+        :return: Google Ads API response, converted to Google Ads Row objects
+        :rtype: list[GoogleAdsRow]
+        """
+        return self._search(client_ids, query, page_size, **kwargs)
+
+    def list_accessible_customers(self) -> List[str]:
+        """
+        Returns resource names of customers directly accessible by the user authenticating the call.
+        The resulting list of customers is based on your OAuth credentials. The request returns a list
+        of all accounts that you are able to act upon directly given your current credentials. This will
+        not necessarily include all accounts within the account hierarchy; rather, it will only include
+        accounts where your authenticated user has been added with admin or other rights in the account.
+
+        ..seealso::
+            https://developers.google.com/google-ads/api/reference/rpc
+
+        :return: List of names of customers
+        """
+        try:
+            accessible_customers = self._get_customer_service.list_accessible_customers()
+            return accessible_customers.resource_names
+        except GoogleAdsException as ex:
+            for error in ex.failure.errors:
+                self.log.error('\tError with message "%s".', error.message)
+                if error.location:
+                    for field_path_element in error.location.field_path_elements:
+                        self.log.error('\t\tOn field: %s', field_path_element.field_name)
+            raise
+
     @cached_property
     def _get_service(self) -> Resource:
         """Connects and authenticates with the Google Ads API using a service account"""
+
+        client = self._get_client
+        return client.get_service("GoogleAdsService", version=self.api_version)
+
+    @cached_property
+    def _get_client(self) -> Resource:
         with NamedTemporaryFile("w", suffix=".json") as secrets_temp:
             self._get_config()
             self._update_config_with_secret(secrets_temp)
             try:
                 client = GoogleAdsClient.load_from_dict(self.google_ads_config)
-                return client.get_service("GoogleAdsService", version=self.api_version)
+                return client
             except GoogleAuthError as e:
                 self.log.error("Google Auth Error: %s", e)
                 raise
@@ -140,7 +217,7 @@ def _update_config_with_secret(self, secrets_temp: IO[str]) -> None:
 
         self.google_ads_config["path_to_private_key_file"] = secrets_temp.name
 
-    def search(
+    def _search(
         self, client_ids: List[str], query: str, page_size: int = 10000, **kwargs
     ) -> List[GoogleAdsRow]:
         """
@@ -157,9 +234,17 @@ def search(
         :rtype: list[GoogleAdsRow]
         """
         service = self._get_service
-        iterators = (
-            service.search(client_id, query=query, page_size=page_size, **kwargs) for client_id in client_ids
-        )
+
+        iterators = []
+        for client_id in client_ids:
+            request = self._get_client.get_type("SearchGoogleAdsRequest")
+            request.customer_id = client_id
+            request.query = query
+            request.page_size = 10000
+
+            iterator = service.search(request=request)
+            iterators.append(iterator)
+
         self.log.info("Fetched Google Ads Iterators")
 
         return self._extract_rows(iterators)
@@ -189,27 +274,3 @@ def _extract_rows(self, iterators: Generator[GRPCIterator, None, None]) -> List[
                     for field_path_element in error.location.field_path_elements:
                         self.log.error("\t\tOn field: %s", field_path_element.field_name)
             raise
-
-    def list_accessible_customers(self) -> List[str]:
-        """
-        Returns resource names of customers directly accessible by the user authenticating the call.
-        The resulting list of customers is based on your OAuth credentials. The request returns a list
-        of all accounts that you are able to act upon directly given your current credentials. This will
-        not necessarily include all accounts within the account hierarchy; rather, it will only include
-        accounts where your authenticated user has been added with admin or other rights in the account.
-
-        ..seealso::
-            https://developers.google.com/google-ads/api/reference/rpc
-
-        :return: List of names of customers
-        """
-        try:
-            accessible_customers = self._get_customer_service.list_accessible_customers()
-            return accessible_customers.resource_names
-        except GoogleAdsException as ex:
-            for error in ex.failure.errors:
-                self.log.error('\tError with message "%s".', error.message)
-                if error.location:
-                    for field_path_element in error.location.field_path_elements:
-                        self.log.error('\t\tOn field: %s', field_path_element.field_name)
-            raise
diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst
@@ -90,7 +90,7 @@ PIP package                             Version required
 ======================================  ===================
 ``apache-airflow``                      ``>=2.1.0``
 ``PyOpenSSL``
-``google-ads``                          ``>=4.0.0,<8.0.0``
+``google-ads``                          ``>=12.0.0``
 ``google-api-core``                     ``>=1.25.1,<2.0.0``
 ``google-api-python-client``            ``>=1.6.0,<2.0.0``
 ``google-auth-httplib2``                ``>=0.0.1``
diff --git a/setup.py b/setup.py
@@ -277,7 +277,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
 ]
 google = [
     'PyOpenSSL',
-    'google-ads>=4.0.0,<8.0.0',
+    'google-ads>=12.0.0',
     'google-api-core>=1.25.1,<2.0.0',
     'google-api-python-client>=1.6.0,<2.0.0',
     'google-auth>=1.0.0,<2.0.0',
diff --git a/tests/providers/google/ads/operators/test_ads.py b/tests/providers/google/ads/operators/test_ads.py
@@ -37,7 +37,7 @@
 
 gcp_conn_id = "gcp_conn_id"
 google_ads_conn_id = "google_ads_conn_id"
-api_version = "v5"
+api_version = "v8"
 
 
 class TestGoogleAdsListAccountsOperator:

Original file line number	Diff line number	Diff line change
`@@ -277,7 +277,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version`
`277`	`277`	`]`
`278`	`278`	`google = [`
`279`	`279`	`'PyOpenSSL',`
`280`		`- 'google-ads>=4.0.0,<8.0.0',`
	`280`	`+ 'google-ads>=12.0.0',`
`281`	`281`	`'google-api-core>=1.25.1,<2.0.0',`
`282`	`282`	`'google-api-python-client>=1.6.0,<2.0.0',`
`283`	`283`	`'google-auth>=1.0.0,<2.0.0',`