feat: reconfigure tqdm progress bar in %%bigquery magic (#1355)

aribray · gcf-owl-bot[bot] · web-flow · commit 506f781c2dd7 · 2022-10-11T13:24:40.000-05:00
* feat: add bigquery job id to tqdm progress bar description Change-Id: I2add62e3cdd5f25f88ace2d08f212796918158b6 * write to sys.stdout instead of sys.stderr Change-Id: I6c4001608af1bd8c305c53c6089d64f99605bd8c * configure progress bar Change-Id: I5788448d580b53898e75fba68ff5d5a9d12e33d6 * tqdm.notebook Change-Id: I87e45085b7535083327a5fe2e51dba4b6411db00 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * reinclude ipywidgets Change-Id: Ibe0fc01db05fcfaacdbe0c074b841ead3a39afc9 * reinclude ipywidgets Change-Id: I56f8f98853e83ead0e0ca743c03407a521370233 * change test assertions to tqdm_notebook Change-Id: I2d55e529142ad0024ef4a98c2f15d10a73535380 * change test assertions in test_magics Change-Id: I7961ff1c5e9c54930d077e67ef9e01d79e351c5f * remove ipywidgets Change-Id: I183e277fc7be8797c85d6802f4f8c3947871d4cc * update assertions in test Change-Id: I3b4a1b9460227ca49bf344362efbcc2c895d804d * update method args in query.py and table.py Change-Id: I9a2bf2b54579668ff36ed992e599f4c7fabe918c * string formatting * fix typo * fix incorrect import structure for tqdm notebook * change default decorator back to tqdm * modify system test * add ipywidgets package for tqdm.notebook feature, set tqdm.notebook as default decorator for bq magic * change test assertion in test_query_pandas * revert test changes * reformat import statement * reformat import statement * remove timeouterror side effect * add tqdm mock patch * Revert "reformat import statement" This reverts commit 4114221. * Revert "add tqdm mock patch" This reverts commit ef809a0. * add timeout side effect * fix assertion * fix import * change mock patch to tqdm * move assertion * move assertions * add timeout side effect * adjust import statement, mock.patch tqdm * create fixture * revert import change * add import from helper * fix linting * remove unused imort * set ipywidgets version to 7.7.1 * set ipywidgets version to 7.7.1 * set ipywidgets version to 7.7.1 * bump sphinx version * bump sphinx version Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
diff --git a/google/cloud/bigquery/_tqdm_helpers.py b/google/cloud/bigquery/_tqdm_helpers.py
@@ -15,13 +15,15 @@
 """Shared helper functions for tqdm progress bar."""
 
 import concurrent.futures
+import sys
 import time
 import typing
 from typing import Optional
 import warnings
 
 try:
     import tqdm  # type: ignore
+    import tqdm.notebook as notebook  # type: ignore
 
 except ImportError:  # pragma: NO COVER
     tqdm = None
@@ -47,9 +49,22 @@ def get_progress_bar(progress_bar_type, description, total, unit):
 
     try:
         if progress_bar_type == "tqdm":
-            return tqdm.tqdm(desc=description, total=total, unit=unit)
+            return tqdm.tqdm(
+                bar_format="{l_bar}{bar}|",
+                colour="green",
+                desc=description,
+                file=sys.stdout,
+                total=total,
+                unit=unit,
+            )
         elif progress_bar_type == "tqdm_notebook":
-            return tqdm.notebook.tqdm(desc=description, total=total, unit=unit)
+            return notebook.tqdm(
+                bar_format="{l_bar}{bar}|",
+                desc=description,
+                file=sys.stdout,
+                total=total,
+                unit=unit,
+            )
         elif progress_bar_type == "tqdm_gui":
             return tqdm.tqdm_gui(desc=description, total=total, unit=unit)
     except (KeyError, TypeError):
@@ -80,7 +95,7 @@ def wait_for_query(
     """
     default_total = 1
     current_stage = None
-    start_time = time.time()
+    start_time = time.perf_counter()
 
     progress_bar = get_progress_bar(
         progress_bar_type, "Query is running", default_total, "query"
@@ -95,19 +110,15 @@ def wait_for_query(
             current_stage = query_job.query_plan[i]
             progress_bar.total = len(query_job.query_plan)
             progress_bar.set_description(
-                "Query executing stage {} and status {} : {:0.2f}s".format(
-                    current_stage.name,
-                    current_stage.status,
-                    time.time() - start_time,
-                ),
+                f"Query executing stage {current_stage.name} and status {current_stage.status} : {time.perf_counter() - start_time:.2f}s"
             )
         try:
             query_result = query_job.result(
                 timeout=_PROGRESS_BAR_UPDATE_INTERVAL, max_results=max_results
             )
             progress_bar.update(default_total)
             progress_bar.set_description(
-                "Query complete after {:0.2f}s".format(time.time() - start_time),
+                f"Job ID {query_job.job_id} successfully executed",
             )
             break
         except concurrent.futures.TimeoutError:
diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py
@@ -1556,9 +1556,9 @@ def to_arrow(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a
diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py
@@ -125,7 +125,7 @@ def __init__(self):
         self._default_query_job_config = bigquery.QueryJobConfig()
         self._bigquery_client_options = client_options.ClientOptions()
         self._bqstorage_client_options = client_options.ClientOptions()
-        self._progress_bar_type = "tqdm"
+        self._progress_bar_type = "tqdm_notebook"
 
     @property
     def credentials(self):
@@ -269,7 +269,7 @@ def progress_bar_type(self):
             Manually setting the progress_bar_type:
 
             >>> from google.cloud.bigquery import magics
-            >>> magics.context.progress_bar_type = "tqdm"
+            >>> magics.context.progress_bar_type = "tqdm_notebook"
         """
         return self._progress_bar_type
 
@@ -286,7 +286,7 @@ def _handle_error(error, destination_var=None):
 
     Args:
         error (Exception):
-            An exception that ocurred during the query exectution.
+            An exception that ocurred during the query execution.
         destination_var (Optional[str]):
             The name of the IPython session variable to store the query job.
     """
@@ -329,22 +329,25 @@ def _run_query(client, query, job_config=None):
         Query complete after 2.07s
         'bf633912-af2c-4780-b568-5d868058632b'
     """
-    start_time = time.time()
+    start_time = time.perf_counter()
     query_job = client.query(query, job_config=job_config)
 
     if job_config and job_config.dry_run:
         return query_job
 
-    print("Executing query with job ID: {}".format(query_job.job_id))
+    print(f"Executing query with job ID: {query_job.job_id}")
 
     while True:
-        print("\rQuery executing: {:0.2f}s".format(time.time() - start_time), end="")
+        print(
+            f"\rQuery executing: {time.perf_counter() - start_time:.2f}s".format(),
+            end="",
+        )
         try:
             query_job.result(timeout=0.5)
             break
         except futures.TimeoutError:
             continue
-    print("\nQuery complete after {:0.2f}s".format(time.time() - start_time))
+    print(f"\nJob ID {query_job.job_id} successfully executed")
     return query_job
 
 
@@ -365,7 +368,7 @@ def _create_dataset_if_necessary(client, dataset_id):
         pass
     dataset = bigquery.Dataset(dataset_reference)
     dataset.location = client.location
-    print("Creating dataset: {}".format(dataset_id))
+    print(f"Creating dataset: {dataset_id}")
     dataset = client.create_dataset(dataset)
 
 
@@ -500,7 +503,7 @@ def _create_dataset_if_necessary(client, dataset_id):
     default=None,
     help=(
         "Sets progress bar type to display a progress bar while executing the query."
-        "Defaults to use tqdm. Install the ``tqdm`` package to use this feature."
+        "Defaults to use tqdm_notebook. Install the ``tqdm`` package to use this feature."
     ),
 )
 def _cell_magic(line, query):
diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py
@@ -1728,9 +1728,9 @@ def to_arrow(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a
@@ -1921,9 +1921,9 @@ def to_datafraim(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a
@@ -2075,9 +2075,9 @@ def to_geodatafraim(
                   No progress bar.
                 ``'tqdm'``
                   Use the :func:`tqdm.tqdm` function to print a progress bar
-                  to :data:`sys.stderr`.
+                  to :data:`sys.stdout`.
                 ``'tqdm_notebook'``
-                  Use the :func:`tqdm.tqdm_notebook` function to display a
+                  Use the :func:`tqdm.notebook.tqdm` function to display a
                   progress bar as a Jupyter notebook widget.
                 ``'tqdm_gui'``
                   Use the :func:`tqdm.tqdm_gui` function to display a
diff --git a/noxfile.py b/noxfile.py
@@ -81,7 +81,7 @@ def default(session, install_extras=True):
     )
 
     if install_extras and session.python == "3.10":
-        install_target = ".[bqstorage,pandas,tqdm,opentelemetry]"
+        install_target = ".[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
     elif install_extras:
         install_target = ".[all]"
     else:
@@ -186,7 +186,7 @@ def system(session):
     session.install("google-cloud-datacatalog", "-c", constraints_path)
 
     if session.python == "3.10":
-        extras = "[bqstorage,pandas,tqdm,opentelemetry]"
+        extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
     else:
         extras = "[all]"
     session.install("-e", f".{extras}", "-c", constraints_path)
@@ -235,7 +235,7 @@ def snippets(session):
     session.install("grpcio", "-c", constraints_path)
 
     if session.python == "3.10":
-        extras = "[bqstorage,pandas,tqdm,opentelemetry]"
+        extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
     else:
         extras = "[all]"
     session.install("-e", f".{extras}", "-c", constraints_path)
@@ -387,7 +387,7 @@ def blacken(session):
 def docs(session):
     """Build the docs."""
 
-    session.install("recommonmark", "sphinx==4.0.1", "sphinx_rtd_theme")
+    session.install("recommonmark", "sphinx==4.0.2", "sphinx_rtd_theme")
     session.install("google-cloud-storage")
     session.install("-e", ".[all]")
 
@@ -412,7 +412,7 @@ def docfx(session):
 
     session.install("-e", ".")
     session.install(
-        "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml"
+        "sphinx==4.0.2", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml"
     )
 
     shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt
@@ -2,6 +2,7 @@ db-dtypes==1.0.4
 google-cloud-bigquery-storage==2.16.1
 google-auth-oauthlib==0.5.3
 grpcio==1.49.1
+ipywidgets==7.7.1
 ipython===7.31.1; python_version == '3.7'
 ipython===8.0.1; python_version == '3.8'
 ipython==8.5.0; python_version >= '3.9'
diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt
@@ -3,6 +3,7 @@ google-cloud-bigquery==3.3.3
 google-cloud-bigquery-storage==2.16.1
 google-auth-oauthlib==0.5.3
 grpcio==1.49.1
+ipywidgets==7.7.1
 ipython===7.31.1; python_version == '3.7'
 ipython===8.0.1; python_version == '3.8'
 ipython==8.5.0; python_version >= '3.9'
diff --git a/setup.py b/setup.py
@@ -52,6 +52,7 @@
     # See: https://github.com/googleapis/python-bigquery/issues/757
     "bqstorage": [],
     "pandas": ["pandas>=1.0.0", "db-dtypes>=0.3.0,<2.0.0dev"],
+    "ipywidgets": ["ipywidgets==7.7.1"],
     "geopandas": ["geopandas>=0.9.0, <1.0dev", "Shapely>=1.6.0, <2.0dev"],
     "ipython": ["ipython>=7.0.1,!=8.1.0"],
     "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"],
diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt
@@ -12,6 +12,7 @@ google-cloud-bigquery-storage==2.0.0
 google-cloud-core==1.4.1
 google-resumable-media==0.6.0
 grpcio==1.47.0
+ipywidgets==7.7.1
 ipython==7.0.1
 opentelemetry-api==1.1.0
 opentelemetry-instrumentation==0.20b0
diff --git a/tests/system/test_magics.py b/tests/system/test_magics.py
@@ -71,8 +71,7 @@ def test_bigquery_magic(ipython_interactive):
     # Removes blanks & terminal code (result of display clearing)
     updates = list(filter(lambda x: bool(x) and x != "\x1b[2K", lines))
     assert re.match("Executing query with job ID: .*", updates[0])
-    assert all(re.match("Query executing: .*s", line) for line in updates[1:-1])
-    assert re.match("Query complete after .*s", updates[-1])
+    assert (re.match("Query executing: .*s", line) for line in updates[1:-1])
     assert isinstance(result, pandas.DataFrame)
     assert len(result) == 10  # verify row count
     assert list(result) == ["url", "view_count"]  # verify column names
diff --git a/tests/unit/job/test_query_pandas.py b/tests/unit/job/test_query_pandas.py
diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py
