From bb1b1e3669738b551e1a345e0f37dfdede656a17 Mon Sep 17 00:00:00 2001 From: TrevorBergeron Date: Tue, 2 Jul 2024 13:36:04 -0700 Subject: [PATCH 1/8] refactor: Add mappings from internal dtypes to bq types (#810) --- bigframes/core/blocks.py | 21 +++++++--------- bigframes/core/schema.py | 6 +++++ bigframes/dtypes.py | 54 ++++++++++++++++++++++++++++++++++------ 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/bigframes/core/blocks.py b/bigframes/core/blocks.py index 379c661179..25acfe2f42 100644 --- a/bigframes/core/blocks.py +++ b/bigframes/core/blocks.py @@ -444,27 +444,24 @@ def _to_dataframe(self, result) -> pd.DataFrame: # Runs strict validations to ensure internal type predictions and ibis are completely in sync # Do not execute these validations outside of testing suite. if "PYTEST_CURRENT_TEST" in os.environ: - self._validate_result_schema(result_dataframe) + self._validate_result_schema(result.schema) return result_dataframe - def _validate_result_schema(self, result_df: pd.DataFrame): + def _validate_result_schema( + self, bq_result_schema: list[bigquery.schema.SchemaField] + ): + actual_schema = tuple(bq_result_schema) ibis_schema = self.expr._compiled_schema internal_schema = self.expr.node.schema - actual_schema = bf_schema.ArraySchema( - tuple( - bf_schema.SchemaItem(name, dtype) # type: ignore - for name, dtype in result_df.dtypes.items() - ) - ) if not bigframes.features.PANDAS_VERSIONS.is_arrow_list_dtype_usable: return - if internal_schema != actual_schema: + if internal_schema.to_bigquery() != actual_schema: raise ValueError( - f"This error should only occur while testing. BigFrames internal schema: {internal_schema} does not match actual schema: {actual_schema}" + f"This error should only occur while testing. BigFrames internal schema: {internal_schema.to_bigquery()} does not match actual schema: {actual_schema}" ) - if ibis_schema != actual_schema: + if ibis_schema.to_bigquery() != actual_schema: raise ValueError( - f"This error should only occur while testing. Ibis schema: {ibis_schema} does not match actual schema: {actual_schema}" + f"This error should only occur while testing. Ibis schema: {ibis_schema.to_bigquery()} does not match actual schema: {actual_schema}" ) def to_arrow( diff --git a/bigframes/core/schema.py b/bigframes/core/schema.py index 30a2a6593a..ee27c6ff30 100644 --- a/bigframes/core/schema.py +++ b/bigframes/core/schema.py @@ -58,6 +58,12 @@ def dtypes(self) -> typing.Tuple[bigframes.dtypes.Dtype, ...]: def _mapping(self) -> typing.Dict[ColumnIdentifierType, bigframes.dtypes.Dtype]: return {item.column: item.dtype for item in self.items} + def to_bigquery(self) -> typing.Tuple[google.cloud.bigquery.SchemaField, ...]: + return tuple( + bigframes.dtypes.convert_to_schema_field(item.column, item.dtype) + for item in self.items + ) + def drop(self, columns: typing.Iterable[str]) -> ArraySchema: return ArraySchema( tuple(item for item in self.items if item.column not in columns) diff --git a/bigframes/dtypes.py b/bigframes/dtypes.py index 160802ded9..563904fbb6 100644 --- a/bigframes/dtypes.py +++ b/bigframes/dtypes.py @@ -70,7 +70,9 @@ class SimpleDtypeInfo: dtype: Dtype arrow_dtype: typing.Optional[pa.DataType] - type_kind: typing.Tuple[str, ...] # Should all correspond to the same db type + type_kind: typing.Tuple[ + str, ... + ] # Should all correspond to the same db type. Put preferred canonical sql type name first logical_bytes: int = ( 8 # this is approximate only, some types are variably sized, also, compression ) @@ -84,20 +86,23 @@ class SimpleDtypeInfo: SimpleDtypeInfo( dtype=INT_DTYPE, arrow_dtype=pa.int64(), - type_kind=("INT64", "INTEGER"), + type_kind=("INTEGER", "INT64"), orderable=True, clusterable=True, ), SimpleDtypeInfo( dtype=FLOAT_DTYPE, arrow_dtype=pa.float64(), - type_kind=("FLOAT64", "FLOAT"), + type_kind=("FLOAT", "FLOAT64"), orderable=True, ), SimpleDtypeInfo( dtype=BOOL_DTYPE, arrow_dtype=pa.bool_(), - type_kind=("BOOL", "BOOLEAN"), + type_kind=( + "BOOLEAN", + "BOOL", + ), logical_bytes=1, orderable=True, clusterable=True, @@ -143,7 +148,7 @@ class SimpleDtypeInfo: SimpleDtypeInfo( dtype=NUMERIC_DTYPE, arrow_dtype=pa.decimal128(38, 9), - type_kind=("NUMERIC",), + type_kind=("NUMERIC", "DECIMAL"), logical_bytes=16, orderable=True, clusterable=True, @@ -151,7 +156,7 @@ class SimpleDtypeInfo: SimpleDtypeInfo( dtype=BIGNUMERIC_DTYPE, arrow_dtype=pa.decimal256(76, 38), - type_kind=("BIGNUMERIC",), + type_kind=("BIGNUMERIC", "BIGDECIMAL"), logical_bytes=32, orderable=True, clusterable=True, @@ -417,6 +422,7 @@ def infer_literal_arrow_type(literal) -> typing.Optional[pa.DataType]: for mapping in SIMPLE_TYPES for type_kind in mapping.type_kind } +_BIGFRAMES_TO_TK = {mapping.dtype: mapping.type_kind[0] for mapping in SIMPLE_TYPES} def convert_schema_field( @@ -440,12 +446,44 @@ def convert_schema_field( if is_repeated: pa_type = pa.list_(bigframes_dtype_to_arrow_dtype(singular_type)) return field.name, pd.ArrowDtype(pa_type) - else: - return field.name, singular_type + return field.name, singular_type else: raise ValueError(f"Cannot handle type: {field.field_type}") +def convert_to_schema_field( + name: str, + bigframes_dtype: Dtype, +) -> google.cloud.bigquery.SchemaField: + if bigframes_dtype in _BIGFRAMES_TO_TK: + return google.cloud.bigquery.SchemaField( + name, _BIGFRAMES_TO_TK[bigframes_dtype] + ) + if isinstance(bigframes_dtype, pd.ArrowDtype): + if pa.types.is_list(bigframes_dtype.pyarrow_dtype): + inner_type = arrow_dtype_to_bigframes_dtype( + bigframes_dtype.pyarrow_dtype.value_type + ) + inner_field = convert_to_schema_field(name, inner_type) + return google.cloud.bigquery.SchemaField( + name, inner_field.field_type, mode="REPEATED", fields=inner_field.fields + ) + if pa.types.is_struct(bigframes_dtype.pyarrow_dtype): + inner_fields: list[pa.Field] = [] + struct_type = typing.cast(pa.StructType, bigframes_dtype.pyarrow_dtype) + for i in range(struct_type.num_fields): + field = struct_type.field(i) + inner_bf_type = arrow_dtype_to_bigframes_dtype(field.type) + inner_fields.append(convert_to_schema_field(field.name, inner_bf_type)) + + return google.cloud.bigquery.SchemaField( + name, "RECORD", fields=inner_fields + ) + raise ValueError( + f"No arrow conversion for {bigframes_dtype}. {constants.FEEDBACK_LINK}" + ) + + def bf_type_from_type_kind( bq_schema: list[google.cloud.bigquery.SchemaField], ) -> typing.Dict[str, Dtype]: From 68b02d5467373b944de9942eb4e26a15e5f2f402 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:44:09 -0400 Subject: [PATCH 2/8] chore: update templated files (#819) Source-Link: https://github.com/googleapis/synthtool/commit/a37f74cd300d1f56d6f28c368d2931f72adee948 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:d3de8a02819f65001effcbd3ea76ce97e9bcff035c7a89457f40f892c87c5b32 Co-authored-by: Owl Bot --- .coveragerc | 2 +- .flake8 | 2 +- .github/.OwlBot.lock.yaml | 4 +- .github/auto-label.yaml | 2 +- .kokoro/docker/docs/Dockerfile | 2 +- .kokoro/populate-secrets.sh | 2 +- .kokoro/publish-docs.sh | 2 +- .kokoro/release.sh | 2 +- .kokoro/requirements.txt | 509 ++++++++++++++------------- .kokoro/test-samples-against-head.sh | 2 +- .kokoro/test-samples-impl.sh | 2 +- .kokoro/test-samples.sh | 2 +- .kokoro/trampoline.sh | 2 +- .kokoro/trampoline_v2.sh | 2 +- .trampolinerc | 2 +- MANIFEST.in | 2 +- docs/conf.py | 2 +- scripts/decrypt-secrets.sh | 2 +- scripts/readme-gen/readme_gen.py | 2 +- 19 files changed, 285 insertions(+), 262 deletions(-) diff --git a/.coveragerc b/.coveragerc index dffe22a6f4..9d801989cb 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.flake8 b/.flake8 index 87f6e408c4..32986c7928 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 81f87c5691..91d742b5b9 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5a4c19d17e597b92d786e569be101e636c9c2817731f80a5adec56b2aa8fe070 -# created: 2024-04-12T11:35:58.922854369Z + digest: sha256:d3de8a02819f65001effcbd3ea76ce97e9bcff035c7a89457f40f892c87c5b32 +# created: 2024-07-03T17:43:00.77142528Z diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml index 8b37ee8971..21786a4eb0 100644 --- a/.github/auto-label.yaml +++ b/.github/auto-label.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index bdaf39fe22..a26ce61930 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh index 6f3972140e..c435402f47 100755 --- a/.kokoro/populate-secrets.sh +++ b/.kokoro/populate-secrets.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC. +# Copyright 2024 Google LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 7700c90ee9..da9ce803dd 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 320ac51271..21a9b558c5 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 51f92b8e12..35ece0e4d2 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -4,21 +4,25 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.1.4 \ - --hash=sha256:72558ba729e4c468572609817226fb0a6e7e9a0a7d477b882be168c0b4a62b94 \ - --hash=sha256:fbe56f8cda08aa9a04b307d8482ea703e96a6a801611acb4be9bf3942017989f +argcomplete==3.4.0 \ + --hash=sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5 \ + --hash=sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f # via nox -attrs==23.1.0 \ - --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ - --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 +attrs==23.2.0 \ + --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ + --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 # via gcp-releasetool -cachetools==5.3.2 \ - --hash=sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2 \ - --hash=sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1 +backports-tarfile==1.2.0 \ + --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ + --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 + # via jaraco-context +cachetools==5.3.3 \ + --hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \ + --hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105 # via google-auth -certifi==2023.7.22 \ - --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ - --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 +certifi==2024.6.2 \ + --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \ + --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56 # via requests cffi==1.16.0 \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -87,90 +91,90 @@ click==8.0.4 \ # -r requirements.in # gcp-docuploader # gcp-releasetool -colorlog==6.7.0 \ - --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ - --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 +colorlog==6.8.2 \ + --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ + --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 # via # gcp-docuploader # nox -cryptography==42.0.5 \ - --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \ - --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \ - --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \ - --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \ - --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \ - --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \ - --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \ - --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \ - --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \ - --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \ - --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \ - --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \ - --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \ - --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \ - --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \ - --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \ - --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \ - --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \ - --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \ - --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \ - --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \ - --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \ - --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \ - --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \ - --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \ - --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \ - --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \ - --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \ - --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \ - --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ - --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ - --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 +cryptography==42.0.8 \ + --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ + --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ + --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ + --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ + --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ + --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ + --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ + --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ + --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ + --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ + --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ + --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ + --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ + --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ + --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ + --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ + --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ + --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ + --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ + --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ + --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ + --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ + --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ + --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ + --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ + --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ + --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ + --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ + --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ + --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ + --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ + --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e # via # -r requirements.in # gcp-releasetool # secretstorage -distlib==0.3.7 \ - --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ - --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 +distlib==0.3.8 \ + --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ + --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 # via virtualenv -docutils==0.20.1 \ - --hash=sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6 \ - --hash=sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b +docutils==0.21.2 \ + --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \ + --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 # via readme-renderer -filelock==3.13.1 \ - --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ - --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c +filelock==3.15.4 \ + --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \ + --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7 # via virtualenv gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==2.0.0 \ - --hash=sha256:3d73480b50ba243f22d7c7ec08b115a30e1c7817c4899781840c26f9c55b8277 \ - --hash=sha256:7aa9fd935ec61e581eb8458ad00823786d91756c25e492f372b2b30962f3c28f +gcp-releasetool==2.0.1 \ + --hash=sha256:34314a910c08e8911d9c965bd44f8f2185c4f556e737d719c33a41f6a610de96 \ + --hash=sha256:b0d5863c6a070702b10883d37c4bdfd74bf930fe417f36c0c965d3b7c779ae62 # via -r requirements.in -google-api-core==2.12.0 \ - --hash=sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553 \ - --hash=sha256:ec6054f7d64ad13b41e43d96f735acbd763b0f3b695dabaa2d579673f6a6e160 +google-api-core==2.19.1 \ + --hash=sha256:f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125 \ + --hash=sha256:f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd # via # google-cloud-core # google-cloud-storage -google-auth==2.23.4 \ - --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ - --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 +google-auth==2.31.0 \ + --hash=sha256:042c4702efa9f7d3c48d3a69341c209381b125faa6dbf3ebe56bc7e40ae05c23 \ + --hash=sha256:87805c36970047247c8afe614d4e3af8eceafc1ebba0c679fe75ddd1d575e871 # via # gcp-releasetool # google-api-core # google-cloud-core # google-cloud-storage -google-cloud-core==2.3.3 \ - --hash=sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb \ - --hash=sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863 +google-cloud-core==2.4.1 \ + --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ + --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via google-cloud-storage -google-cloud-storage==2.13.0 \ - --hash=sha256:ab0bf2e1780a1b74cf17fccb13788070b729f50c252f0c94ada2aae0ca95437d \ - --hash=sha256:f62dc4c7b6cd4360d072e3deb28035fbdad491ac3d9b0b1815a12daea10f37c7 +google-cloud-storage==2.17.0 \ + --hash=sha256:49378abff54ef656b52dca5ef0f2eba9aa83dc2b2c72c78714b03a1a95fe9388 \ + --hash=sha256:5b393bc766b7a3bc6f5407b9e665b2450d36282614b7945e570b3480a456d1e1 # via gcp-docuploader google-crc32c==1.5.0 \ --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ @@ -244,28 +248,36 @@ google-crc32c==1.5.0 \ # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 \ - --hash=sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7 \ - --hash=sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b +google-resumable-media==2.7.1 \ + --hash=sha256:103ebc4ba331ab1bfdac0250f8033627a2cd7cde09e7ccff9181e31ba4315b2c \ + --hash=sha256:eae451a7b2e2cdbaaa0fd2eb00cc8a1ee5e95e16b55597359cbc3d27d7d90e33 # via google-cloud-storage -googleapis-common-protos==1.61.0 \ - --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ - --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b +googleapis-common-protos==1.63.2 \ + --hash=sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945 \ + --hash=sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87 # via google-api-core idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via requests -importlib-metadata==6.8.0 \ - --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \ - --hash=sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743 +importlib-metadata==8.0.0 \ + --hash=sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f \ + --hash=sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812 # via # -r requirements.in # keyring # twine -jaraco-classes==3.3.0 \ - --hash=sha256:10afa92b6743f25c0cf5f37c6bb6e18e2c5bb84a16527ccfc0040ea377e7aaeb \ - --hash=sha256:c063dd08e89217cee02c8d5e5ec560f2c8ce6cdc2fcdc2e68f7b2e5547ed3621 +jaraco-classes==3.4.0 \ + --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 + # via keyring +jaraco-context==5.3.0 \ + --hash=sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266 \ + --hash=sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2 + # via keyring +jaraco-functools==4.0.1 \ + --hash=sha256:3b24ccb921d6b593bdceb56ce14799204f473976e2a9d4b15b04d0f2c2326664 \ + --hash=sha256:d33fa765374c0611b52f8b3a795f8900869aa88c84769d4d1746cd68fb28c3e8 # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ @@ -273,13 +285,13 @@ jeepney==0.8.0 \ # via # keyring # secretstorage -jinja2==3.1.3 \ - --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ - --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90 +jinja2==3.1.4 \ + --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ + --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d # via gcp-releasetool -keyring==24.2.0 \ - --hash=sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6 \ - --hash=sha256:ca0746a19ec421219f4d713f848fa297a661a8a8c1504867e55bfb5e09091509 +keyring==25.2.1 \ + --hash=sha256:2458681cdefc0dbc0b7eb6cf75d0b98e59f9ad9b2d4edd319d18f68bdca95e50 \ + --hash=sha256:daaffd42dbda25ddafb1ad5fec4024e5bbcfe424597ca1ca452b299861e49f1b # via # gcp-releasetool # twine @@ -287,146 +299,153 @@ markdown-it-py==3.0.0 \ --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb # via rich -markupsafe==2.1.3 \ - --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ - --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ - --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ - --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ - --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ - --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ - --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ - --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ - --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ - --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ - --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ - --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ - --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ - --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ - --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ - --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ - --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ - --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ - --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ - --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ - --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ - --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ - --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ - --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ - --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ - --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ - --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ - --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ - --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ - --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ - --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ - --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ - --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ - --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ - --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ - --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ - --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ - --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ - --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ - --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ - --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ - --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ - --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ - --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ - --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ - --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ - --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ - --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ - --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ - --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ - --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ - --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ - --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ - --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ - --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ - --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ - --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ - --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ - --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ - --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 +markupsafe==2.1.5 \ + --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ + --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ + --hash=sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f \ + --hash=sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3 \ + --hash=sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532 \ + --hash=sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f \ + --hash=sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617 \ + --hash=sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df \ + --hash=sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4 \ + --hash=sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906 \ + --hash=sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f \ + --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ + --hash=sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8 \ + --hash=sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371 \ + --hash=sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2 \ + --hash=sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465 \ + --hash=sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52 \ + --hash=sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6 \ + --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ + --hash=sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad \ + --hash=sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2 \ + --hash=sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0 \ + --hash=sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029 \ + --hash=sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f \ + --hash=sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a \ + --hash=sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced \ + --hash=sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5 \ + --hash=sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c \ + --hash=sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf \ + --hash=sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9 \ + --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ + --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ + --hash=sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3 \ + --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ + --hash=sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46 \ + --hash=sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc \ + --hash=sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a \ + --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ + --hash=sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900 \ + --hash=sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5 \ + --hash=sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea \ + --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ + --hash=sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5 \ + --hash=sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e \ + --hash=sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a \ + --hash=sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f \ + --hash=sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50 \ + --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ + --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ + --hash=sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4 \ + --hash=sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff \ + --hash=sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2 \ + --hash=sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46 \ + --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ + --hash=sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf \ + --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 \ + --hash=sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5 \ + --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ + --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ + --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 # via jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -more-itertools==10.1.0 \ - --hash=sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a \ - --hash=sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6 - # via jaraco-classes -nh3==0.2.14 \ - --hash=sha256:116c9515937f94f0057ef50ebcbcc10600860065953ba56f14473ff706371873 \ - --hash=sha256:18415df36db9b001f71a42a3a5395db79cf23d556996090d293764436e98e8ad \ - --hash=sha256:203cac86e313cf6486704d0ec620a992c8bc164c86d3a4fd3d761dd552d839b5 \ - --hash=sha256:2b0be5c792bd43d0abef8ca39dd8acb3c0611052ce466d0401d51ea0d9aa7525 \ - --hash=sha256:377aaf6a9e7c63962f367158d808c6a1344e2b4f83d071c43fbd631b75c4f0b2 \ - --hash=sha256:525846c56c2bcd376f5eaee76063ebf33cf1e620c1498b2a40107f60cfc6054e \ - --hash=sha256:5529a3bf99402c34056576d80ae5547123f1078da76aa99e8ed79e44fa67282d \ - --hash=sha256:7771d43222b639a4cd9e341f870cee336b9d886de1ad9bec8dddab22fe1de450 \ - --hash=sha256:88c753efbcdfc2644a5012938c6b9753f1c64a5723a67f0301ca43e7b85dcf0e \ - --hash=sha256:93a943cfd3e33bd03f77b97baa11990148687877b74193bf777956b67054dcc6 \ - --hash=sha256:9be2f68fb9a40d8440cbf34cbf40758aa7f6093160bfc7fb018cce8e424f0c3a \ - --hash=sha256:a0c509894fd4dccdff557068e5074999ae3b75f4c5a2d6fb5415e782e25679c4 \ - --hash=sha256:ac8056e937f264995a82bf0053ca898a1cb1c9efc7cd68fa07fe0060734df7e4 \ - --hash=sha256:aed56a86daa43966dd790ba86d4b810b219f75b4bb737461b6886ce2bde38fd6 \ - --hash=sha256:e8986f1dd3221d1e741fda0a12eaa4a273f1d80a35e31a1ffe579e7c621d069e \ - --hash=sha256:f99212a81c62b5f22f9e7c3e347aa00491114a5647e1f13bbebd79c3e5f08d75 +more-itertools==10.3.0 \ + --hash=sha256:e5d93ef411224fbcef366a6e8ddc4c5781bc6359d43412a65dd5964e46111463 \ + --hash=sha256:ea6a02e24a9161e51faad17a8782b92a0df82c12c1c8886fec7f0c3fa1a1b320 + # via + # jaraco-classes + # jaraco-functools +nh3==0.2.17 \ + --hash=sha256:0316c25b76289cf23be6b66c77d3608a4fdf537b35426280032f432f14291b9a \ + --hash=sha256:1a814dd7bba1cb0aba5bcb9bebcc88fd801b63e21e2450ae6c52d3b3336bc911 \ + --hash=sha256:1aa52a7def528297f256de0844e8dd680ee279e79583c76d6fa73a978186ddfb \ + --hash=sha256:22c26e20acbb253a5bdd33d432a326d18508a910e4dcf9a3316179860d53345a \ + --hash=sha256:40015514022af31975c0b3bca4014634fa13cb5dc4dbcbc00570acc781316dcc \ + --hash=sha256:40d0741a19c3d645e54efba71cb0d8c475b59135c1e3c580f879ad5514cbf028 \ + --hash=sha256:551672fd71d06cd828e282abdb810d1be24e1abb7ae2543a8fa36a71c1006fe9 \ + --hash=sha256:66f17d78826096291bd264f260213d2b3905e3c7fae6dfc5337d49429f1dc9f3 \ + --hash=sha256:85cdbcca8ef10733bd31f931956f7fbb85145a4d11ab9e6742bbf44d88b7e351 \ + --hash=sha256:a3f55fabe29164ba6026b5ad5c3151c314d136fd67415a17660b4aaddacf1b10 \ + --hash=sha256:b4427ef0d2dfdec10b641ed0bdaf17957eb625b2ec0ea9329b3d28806c153d71 \ + --hash=sha256:ba73a2f8d3a1b966e9cdba7b211779ad8a2561d2dba9674b8a19ed817923f65f \ + --hash=sha256:c21bac1a7245cbd88c0b0e4a420221b7bfa838a2814ee5bb924e9c2f10a1120b \ + --hash=sha256:c551eb2a3876e8ff2ac63dff1585236ed5dfec5ffd82216a7a174f7c5082a78a \ + --hash=sha256:c790769152308421283679a142dbdb3d1c46c79c823008ecea8e8141db1a2062 \ + --hash=sha256:d7a25fd8c86657f5d9d576268e3b3767c5cd4f42867c9383618be8517f0f022a # via readme-renderer -nox==2023.4.22 \ - --hash=sha256:0b1adc619c58ab4fa57d6ab2e7823fe47a32e70202f287d78474adcc7bda1891 \ - --hash=sha256:46c0560b0dc609d7d967dc99e22cb463d3c4caf54a5fda735d6c11b5177e3a9f +nox==2024.4.15 \ + --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ + --hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f # via -r requirements.in -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via # gcp-releasetool # nox -pkginfo==1.9.6 \ - --hash=sha256:4b7a555a6d5a22169fcc9cf7bfd78d296b0361adad412a346c1226849af5e546 \ - --hash=sha256:8fd5896e8718a4372f0ea9cc9d96f6417c9b986e23a4d116dda26b62cc29d046 +pkginfo==1.10.0 \ + --hash=sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297 \ + --hash=sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097 # via twine -platformdirs==3.11.0 \ - --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ - --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e +platformdirs==4.2.2 \ + --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \ + --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3 # via virtualenv -protobuf==4.25.3 \ - --hash=sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4 \ - --hash=sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8 \ - --hash=sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c \ - --hash=sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d \ - --hash=sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4 \ - --hash=sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa \ - --hash=sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c \ - --hash=sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019 \ - --hash=sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9 \ - --hash=sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c \ - --hash=sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2 +proto-plus==1.24.0 \ + --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \ + --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12 + # via google-api-core +protobuf==5.27.2 \ + --hash=sha256:0e341109c609749d501986b835f667c6e1e24531096cff9d34ae411595e26505 \ + --hash=sha256:176c12b1f1c880bf7a76d9f7c75822b6a2bc3db2d28baa4d300e8ce4cde7409b \ + --hash=sha256:354d84fac2b0d76062e9b3221f4abbbacdfd2a4d8af36bab0474f3a0bb30ab38 \ + --hash=sha256:4fadd8d83e1992eed0248bc50a4a6361dc31bcccc84388c54c86e530b7f58863 \ + --hash=sha256:54330f07e4949d09614707c48b06d1a22f8ffb5763c159efd5c0928326a91470 \ + --hash=sha256:610e700f02469c4a997e58e328cac6f305f649826853813177e6290416e846c6 \ + --hash=sha256:7fc3add9e6003e026da5fc9e59b131b8f22b428b991ccd53e2af8071687b4fce \ + --hash=sha256:9e8f199bf7f97bd7ecebffcae45ebf9527603549b2b562df0fbc6d4d688f14ca \ + --hash=sha256:a109916aaac42bff84702fb5187f3edadbc7c97fc2c99c5ff81dd15dcce0d1e5 \ + --hash=sha256:b848dbe1d57ed7c191dfc4ea64b8b004a3f9ece4bf4d0d80a367b76df20bf36e \ + --hash=sha256:f3ecdef226b9af856075f28227ff2c90ce3a594d092c39bee5513573f25e2714 # via # gcp-docuploader # gcp-releasetool # google-api-core # googleapis-common-protos -pyasn1==0.5.0 \ - --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ - --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde + # proto-plus +pyasn1==0.6.0 \ + --hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \ + --hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 \ - --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ - --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d +pyasn1-modules==0.4.0 \ + --hash=sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6 \ + --hash=sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b # via google-auth -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 +pycparser==2.22 \ + --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ + --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc # via cffi -pygments==2.16.1 \ - --hash=sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692 \ - --hash=sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29 +pygments==2.18.0 \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via # readme-renderer # rich @@ -434,20 +453,20 @@ pyjwt==2.8.0 \ --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 # via gcp-releasetool -pyperclip==1.8.2 \ - --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 +pyperclip==1.9.0 \ + --hash=sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310 # via gcp-releasetool -python-dateutil==2.8.2 \ - --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ - --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via gcp-releasetool -readme-renderer==42.0 \ - --hash=sha256:13d039515c1f24de668e2c93f2e877b9dbe6c6c32328b90a40a49d8b2b85f36d \ - --hash=sha256:2d55489f83be4992fe4454939d1a051c33edbab778e82761d060c9fc6b308cd1 +readme-renderer==43.0 \ + --hash=sha256:1818dd28140813509eeed8d62687f7cd4f7bad90d4db586001c5dc09d4fde311 \ + --hash=sha256:19db308d86ecd60e5affa3b2a98f017af384678c63c88e5d4556a380e674f3f9 # via twine -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via # gcp-releasetool # google-api-core @@ -462,9 +481,9 @@ rfc3986==2.0.0 \ --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c # via twine -rich==13.6.0 \ - --hash=sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245 \ - --hash=sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef +rich==13.7.1 \ + --hash=sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222 \ + --hash=sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432 # via twine rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -480,35 +499,39 @@ six==1.16.0 \ # via # gcp-docuploader # python-dateutil -twine==4.0.2 \ - --hash=sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8 \ - --hash=sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8 +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via nox +twine==5.1.1 \ + --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ + --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db # via -r requirements.in -typing-extensions==4.8.0 \ - --hash=sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0 \ - --hash=sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via -r requirements.in -urllib3==2.0.7 \ - --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ - --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e +urllib3==2.2.2 \ + --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ + --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 # via # requests # twine -virtualenv==20.24.6 \ - --hash=sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af \ - --hash=sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381 +virtualenv==20.26.3 \ + --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ + --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589 # via nox -wheel==0.41.3 \ - --hash=sha256:488609bc63a29322326e05560731bf7bfea8e48ad646e1f5e40d366607de0942 \ - --hash=sha256:4d4987ce51a49370ea65c0bfd2234e8ce80a12780820d9dc462597a6e60d0841 +wheel==0.43.0 \ + --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ + --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81 # via -r requirements.in -zipp==3.17.0 \ - --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \ - --hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0 +zipp==3.19.2 \ + --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ + --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==69.2.0 \ - --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \ - --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c +setuptools==70.2.0 \ + --hash=sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05 \ + --hash=sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1 # via -r requirements.in diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh index 63ac41dfae..e9d8bd79a6 100755 --- a/.kokoro/test-samples-against-head.sh +++ b/.kokoro/test-samples-against-head.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 5a0f5fab6a..55910c8ba1 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index 50b35a48c1..7933d82014 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh index d85b1f2676..48f7969970 100755 --- a/.kokoro/trampoline.sh +++ b/.kokoro/trampoline.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 59a7cf3a93..35fa529231 100755 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.trampolinerc b/.trampolinerc index a7dfeb42c6..0080152373 100644 --- a/.trampolinerc +++ b/.trampolinerc @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/MANIFEST.in b/MANIFEST.in index 02b1f4ba4b..16a933a629 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/docs/conf.py b/docs/conf.py index af8c5efda8..23ec7a6b36 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh index 0018b421dd..120b0ddc43 100755 --- a/scripts/decrypt-secrets.sh +++ b/scripts/decrypt-secrets.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2023 Google LLC All rights reserved. +# Copyright 2024 Google LLC All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py index 1acc119835..8f5e248a0d 100644 --- a/scripts/readme-gen/readme_gen.py +++ b/scripts/readme-gen/readme_gen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From bca4ee0c3e7f516588b104655901033aa20b19f1 Mon Sep 17 00:00:00 2001 From: Shobhit Singh Date: Wed, 3 Jul 2024 15:17:26 -0700 Subject: [PATCH 3/8] test: add notebook with remote function use cases (#808) * test: add notebook with remote function usecases * use reuse=False to avoid concurrency conflicts during testing * decorate the notebook with comments * update comment * use aliased import for demo purpose * alias the outside import for demo --- .../remote_function_usecases.ipynb | 1408 +++++++++++++++++ 1 file changed, 1408 insertions(+) create mode 100644 notebooks/remote_functions/remote_function_usecases.ipynb diff --git a/notebooks/remote_functions/remote_function_usecases.ipynb b/notebooks/remote_functions/remote_function_usecases.ipynb new file mode 100644 index 0000000000..3d7ae3e8c7 --- /dev/null +++ b/notebooks/remote_functions/remote_function_usecases.ipynb @@ -0,0 +1,1408 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright 2023 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "id": "Y6QAttCqqMM0" + }, + "outputs": [], + "source": [ + "import bigframes.pandas as bpd" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 296 + }, + "id": "xraJ9RRzsvel", + "outputId": "6e3308cf-8de0-4b89-9128-4c6ddf3598c0" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/shobs/code/bigframes/venv/lib/python3.10/site-packages/IPython/core/interactiveshell.py:3550: UserWarning: Reading cached table from 2024-06-28 02:49:31.716256+00:00 to avoid incompatibilies with previous reads of this table. To read the latest version, set `use_cache=False` or close the current session with Session.close() or bigframes.pandas.close_session().\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + }, + { + "data": { + "text/html": [ + "Query job f72cda67-2a96-4cd2-a624-591c0d540fc9 is DONE. 582.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 65cf6ca3-73f0-49e6-84a8-1ff79af6ec75 is DONE. 82.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutes
50RaysRangers181
72PhilliesPirates192
89MarinersBlue Jays183
351AstrosAngels212
382RoyalsYankees259
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes\n", + "50 Rays Rangers 181\n", + "72 Phillies Pirates 192\n", + "89 Mariners Blue Jays 183\n", + "351 Astros Angels 212\n", + "382 Royals Yankees 259" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = bpd.read_gbq(\"bigquery-public-data.baseball.schedules\")[[\"homeTeamName\", \"awayTeamName\", \"duration_minutes\"]]\n", + "df.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Notes\n", + "\n", + "* The API reference documentation for the `remote_function` can be found at\n", + " https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.session.Session#bigframes_session_Session_remote_function\n", + "\n", + "* More code samples for `remote_function` can be found in the BigQuery\n", + " DataFrames API reference documentation, e.g.\n", + " * https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.series.Series#bigframes_series_Series_apply\n", + " * https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.dataframe.DataFrame#bigframes_dataframe_DataFrame_map\n", + " * https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.dataframe.DataFrame#bigframes_dataframe_DataFrame_apply\n", + "\n", + "* The following examples are only for the purpose of demonstrating\n", + "`remote_function` usage. They are not necessarily the best way to achieve the\n", + "end result.\n", + "\n", + "* In the examples in this notebook we are using `reuse=False` just as a caution\n", + " to avoid concurrent runs of this notebook in the same google cloud project\n", + " stepping over each other's remote function deployment. It may not be neccesary\n", + " in a simple use case." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Pt4mWYE1p5o8" + }, + "source": [ + "# Self-contained function\n", + "\n", + "Let's consider a scenario where we want to categorize the matches as short,\n", + "medium or long duration based on the `duration_minutes` column." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "VoCPBJ-ZpyeG", + "outputId": "19351206-116e-4da2-8ff0-f288b7745b27" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job f039d478-8dc4-4b60-8eda-179955e06586 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-862150459da5240a6df1ce01c59b32d8-em4ibov0' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_862150459da5240a6df1ce01c59b32d8_em4ibov0'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " if duration_minutes < 90:\n", + " return \"short\"\n", + " elif duration_minutes < 180:\n", + " return \"medium\"\n", + " else:\n", + " return \"long\"\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "oXgDB70Lp5cG", + "outputId": "c08aade0-8b03-425b-fc26-deafd89275a4" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 23e95831-d913-4d2b-97f6-588fc7967455 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job bb8b3d13-a521-4d45-b4c8-5686c944a9f2 is DONE. 157.2 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 2a4653f5-cc6b-4279-a45e-40f0f97090a7 is DONE. 98.8 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels132medium
2365AthleticsAngels134medium
1977AthleticsAngels139medium
554CubsAngels142medium
654AstrosAngels143medium
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 medium\n", + "2365 Athletics Angels 134 medium\n", + "1977 Athletics Angels 139 medium\n", + "554 Cubs Angels 142 medium\n", + "654 Astros Angels 143 medium" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zTaNSVmuzEkc" + }, + "source": [ + "# Function referring to variables outside the function body\n", + "\n", + "Let's consider a slight variation of the earlier example where the labels for\n", + "the short, medium and long duration matches are defined outside the function\n", + "body. They would be captured at the time of `remote_function` deployment and\n", + "any change in their values in the notebook after the deployment will not\n", + "automatically propagate to the `remote_function`." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "id": "2UEmTbu4znyS" + }, + "outputs": [], + "source": [ + "DURATION_CATEGORY_SHORT = \"S\"\n", + "DURATION_CATEGORY_MEDIUM = \"M\"\n", + "DURATION_CATEGORY_LONG = \"L\"" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "G-73kpmrznHn", + "outputId": "b5923b7c-d412-43bf-9a20-3946154df81a" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 5d914fde-81ec-46eb-9219-9822f77dd9a2 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-f3231b74ec807496f4894218d5d40ed5-688mx7hi' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_f3231b74ec807496f4894218d5d40ed5_688mx7hi'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " if duration_minutes < 90:\n", + " return DURATION_CATEGORY_SHORT\n", + " elif duration_minutes < 180:\n", + " return DURATION_CATEGORY_MEDIUM\n", + " else:\n", + " return DURATION_CATEGORY_LONG\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "DWHKsfF-z7rL", + "outputId": "c736b57f-1fcb-464a-f725-eb203265ddc2" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job b0b39944-1e69-4185-97ba-985178ee241f is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 90d99515-eb5e-4bcd-bce5-292eea09770e is DONE. 147.7 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job eb31d033-c871-49c5-a75e-4427e376516f is DONE. 89.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels132M
2365AthleticsAngels134M
1977AthleticsAngels139M
554CubsAngels142M
654AstrosAngels143M
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 M\n", + "2365 Athletics Angels 134 M\n", + "1977 Athletics Angels 139 M\n", + "554 Cubs Angels 142 M\n", + "654 Astros Angels 143 M" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J-1BIasNzKil" + }, + "source": [ + "# Function referring to imports (built-in) outside the function body\n", + "\n", + "Let's consider a scenario in which we want to categorize the matches in terms of\n", + "hour buckets. E.g. a match finishing in 0-60 minutes would be in 1h category,\n", + "61-120 minutes in 2h category and so on. The function itself makes use of the\n", + "`math` module (a built-in module in a standard python installation) which\n", + "happens to be imported outside the function body, let's say in one of the\n", + "previous cells. For the demo purpose we have aliased the import to `mymath`, but\n", + "it is not necessary.\n", + "\n", + "Later in the notebook we will see another example with a third-party module." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "id": "zlQfhcW41uzM" + }, + "outputs": [], + "source": [ + "import math as mymath" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "ktADchck2mh4", + "outputId": "9aed6aea-b361-4414-a0f6-8873e8291090" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 2895676f-d15c-40fd-8cf2-3a0436291e6b is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-9b20b0257558a42da610d8998022c25e-7k62x9l6' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_9b20b0257558a42da610d8998022c25e_7k62x9l6'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " duration_hours = mymath.ceil(duration_minutes / 60)\n", + " return f\"{duration_hours}h\"\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "ywAtZlJU3GoB", + "outputId": "d3c93a31-3367-4ccf-bdf7-62d5bbff4461" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 4efda755-2f54-4477-b48a-4a424c888559 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job a8992776-c2e8-4c3e-ab75-dfc01c5de89f is DONE. 150.1 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 3ea299b0-27ad-432b-8dbf-81da3aae884f is DONE. 91.7 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels1323h
2365AthleticsAngels1343h
1977AthleticsAngels1393h
554CubsAngels1423h
654AstrosAngels1433h
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 3h\n", + "2365 Athletics Angels 134 3h\n", + "1977 Athletics Angels 139 3h\n", + "554 Cubs Angels 142 3h\n", + "654 Astros Angels 143 3h" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WO0FH7Bm3OxR" + }, + "source": [ + "# Function referring to another function outside the function body\n", + "\n", + "In this example let's create a `remote_function` from a function\n", + "`duration_category` which depends upon another function `get_hour_ceiling`,\n", + "which further depends on another function `get_minutes_in_hour`. This dependency\n", + "chain could be even longer in a real world example. The behaviors of the\n", + "dependencies would be captured at the time of the remote function\n", + "deployment.\n", + "\n", + "Please ntoe that any changes in those functions in the notebook after the\n", + "deployment would not automatically propagate to the remote function." + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "id": "0G91fWiF3pKg" + }, + "outputs": [], + "source": [ + "import math\n", + "\n", + "def get_minutes_in_hour():\n", + " return 60\n", + "\n", + "def get_hour_ceiling(minutes):\n", + " return math.ceil(minutes / get_minutes_in_hour())" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "lQrC8T2031EJ", + "outputId": "420e7c3d-54cb-4814-f973-c7678be61caa" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 411853db-bf83-4df8-af78-55b1ceb39cb1 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-b54aa0aa752af6a3bd6d9d529dac373b-h4lgpy4y' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_b54aa0aa752af6a3bd6d9d529dac373b_h4lgpy4y'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False)\n", + "def duration_category(duration_minutes: int) -> str:\n", + " duration_hours = get_hour_ceiling(duration_minutes)\n", + " return f\"{duration_hours} hrs\"\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "GVyrihii4EFG", + "outputId": "e979b649-4ed4-4b82-e814-54180420e3fc" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job d04abfa5-e2f2-4936-a708-ed97ef429df3 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 2fc4edf0-7a86-4532-b8fb-bd3f5d153dcb is DONE. 157.4 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job f7e6e18c-70d7-4b4e-926a-03b3a1abd1fe is DONE. 99.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels1323 hrs
2365AthleticsAngels1343 hrs
1977AthleticsAngels1393 hrs
554CubsAngels1423 hrs
654AstrosAngels1433 hrs
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 3 hrs\n", + "2365 Athletics Angels 134 3 hrs\n", + "1977 Athletics Angels 139 3 hrs\n", + "554 Cubs Angels 142 3 hrs\n", + "654 Astros Angels 143 3 hrs" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Uu7SOoT94vSP" + }, + "source": [ + "# Function requiring external packages\n", + "\n", + "In this example let's say we want to redact the `homeTeamName` values, and we\n", + "choose to use a third party library `cryptography`. Any third party dependencies\n", + "can be specified in [pip format](https://pip.pypa.io/en/stable/reference/requirements-file-format/)\n", + "(with or without version number) as a list via the `packages` parameter." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "3EUEyNcW41_l", + "outputId": "2d09d60f-da1a-4eab-86d3-0e62390a360c" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job c674e7b7-2349-4317-8f08-8bfd9aa99785 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "@bpd.remote_function(reuse=False, packages=[\"cryptography\"])\n", + "def get_hash(input: str) -> str:\n", + " from cryptography.fernet import Fernet\n", + "\n", + " # handle missing value\n", + " if input is None:\n", + " input = \"\"\n", + "\n", + " key = Fernet.generate_key()\n", + " f = Fernet(key)\n", + " return f.encrypt(input.encode()).decode()" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 258 + }, + "id": "OX1Hl7bR5uyd", + "outputId": "8ac3bf28-d16d-438b-b636-74ef2371715f" + }, + "outputs": [ + { + "data": { + "text/html": [ + "Query job eb9384c9-de7d-4232-bdca-94b61b50ff89 is DONE. 60.5 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 11a736a5-96d1-4e62-90e2-576156131a94 is DONE. 388.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job c66a9ad1-60f7-4af1-ad7c-65e4eecbb035 is DONE. 330.0 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minuteshomeTeamNameRedacted
719AstrosAngels180gAAAAABmflbKCFygsmoTzFkUCObFSBJG29Ksk8HEtk82ib...
2295AstrosAngels204gAAAAABmflbKv-XzIxcNS92RO4fXYIAwA0kGWsAy-tI5fm...
1126AstrosAngels176gAAAAABmflbJdjgpqnfvmklU7Zg3NJUqlTMYMs44dLEkwg...
294AstrosAngels189gAAAAABmflbKmfBh4P3FnwyiIpVFek9TzF4GzwP_5rQmkv...
351AstrosAngels212gAAAAABmflbJ_mzqao9i7BtoYlMpb6y3bV3x7-cYuWGxsT...
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes \\\n", + "719 Astros Angels 180 \n", + "2295 Astros Angels 204 \n", + "1126 Astros Angels 176 \n", + "294 Astros Angels 189 \n", + "351 Astros Angels 212 \n", + "\n", + " homeTeamNameRedacted \n", + "719 gAAAAABmflbKCFygsmoTzFkUCObFSBJG29Ksk8HEtk82ib... \n", + "2295 gAAAAABmflbKv-XzIxcNS92RO4fXYIAwA0kGWsAy-tI5fm... \n", + "1126 gAAAAABmflbJdjgpqnfvmklU7Zg3NJUqlTMYMs44dLEkwg... \n", + "294 gAAAAABmflbKmfBh4P3FnwyiIpVFek9TzF4GzwP_5rQmkv... \n", + "351 gAAAAABmflbJ_mzqao9i7BtoYlMpb6y3bV3x7-cYuWGxsT... " + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(homeTeamNameRedacted=df[\"homeTeamName\"].apply(get_hash))\n", + "df1.peek()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Function referring to imports (third-party) outside the function body\n", + "\n", + "In this scenario the function depends on a third party library and the module\n", + "from the third party library used in the function is imported outside the\n", + "function body in a previous cell. Below is such an example where the third-party\n", + "dependency is `humanize` and its module of the same name is imported outside the\n", + "function body." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "import datetime as dt\n", + "import humanize" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Query job 21b054a9-8fb2-418f-a17b-effdf5aba9b5 is DONE. 0 Bytes processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created cloud function 'projects/bigframes-dev/locations/us-central1/functions/bigframes-0879f72acd9b8ede460b69c5a8cc0dcb-edxlst27' and BQ remote function 'bigframes-dev._1b6c31ff1bcd5d2f6d86833cf8268317f1b12d57.bigframes_0879f72acd9b8ede460b69c5a8cc0dcb_edxlst27'.\n" + ] + } + ], + "source": [ + "@bpd.remote_function(reuse=False, packages=[\"humanize\"])\n", + "def duration_category(duration_minutes: int) -> str:\n", + " timedelta = dt.timedelta(minutes=duration_minutes)\n", + " return humanize.naturaldelta(timedelta)\n", + "\n", + "print(f\"Created cloud function '{duration_category.bigframes_cloud_function}' and BQ remote function '{duration_category.bigframes_remote_function}'.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Query job d67b7cb9-9813-4863-99d1-01cf45ab4949 is DONE. 58.3 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 579ba853-a7b8-49df-9539-bf22f08d2370 is DONE. 162.2 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Query job 72f9eb5d-1c1a-4ce8-8f2f-1f5a8f7cec99 is DONE. 103.9 kB processed. Open Job" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
homeTeamNameawayTeamNameduration_minutesduration_cat
1911DodgersAngels1322 hours
2365AthleticsAngels1342 hours
1977AthleticsAngels1392 hours
554CubsAngels1422 hours
654AstrosAngels1432 hours
\n", + "
" + ], + "text/plain": [ + " homeTeamName awayTeamName duration_minutes duration_cat\n", + "1911 Dodgers Angels 132 2 hours\n", + "2365 Athletics Angels 134 2 hours\n", + "1977 Athletics Angels 139 2 hours\n", + "554 Cubs Angels 142 2 hours\n", + "654 Astros Angels 143 2 hours" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1 = df.assign(duration_cat=df[\"duration_minutes\"].apply(duration_category))\n", + "df1.peek()" + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From e22801047440eefef9c4acc1731ec7c5c1a62b88 Mon Sep 17 00:00:00 2001 From: TrevorBergeron Date: Wed, 3 Jul 2024 16:08:17 -0700 Subject: [PATCH 4/8] refactor: Create ordering base class to represent partial ordering (#814) --- bigframes/core/__init__.py | 4 +- bigframes/core/compile/api.py | 2 +- bigframes/core/compile/compiled.py | 23 ++-- bigframes/core/compile/compiler.py | 2 +- bigframes/core/compile/concat.py | 4 +- bigframes/core/compile/default_ordering.py | 2 +- bigframes/core/compile/single_column.py | 34 +----- bigframes/core/nodes.py | 2 +- bigframes/core/ordering.py | 117 +++++++++++++++++---- bigframes/session/__init__.py | 2 +- 10 files changed, 116 insertions(+), 76 deletions(-) diff --git a/bigframes/core/__init__.py b/bigframes/core/__init__.py index 00a36b9c05..9b858046bc 100644 --- a/bigframes/core/__init__.py +++ b/bigframes/core/__init__.py @@ -78,7 +78,7 @@ def from_cached( cls, original: ArrayValue, table: google.cloud.bigquery.Table, - ordering: orderings.ExpressionOrdering, + ordering: orderings.TotalOrdering, ): node = nodes.CachedTableNode( original_node=original.node, @@ -147,7 +147,7 @@ def _compiled_schema(self) -> schemata.ArraySchema: def as_cached( self: ArrayValue, cache_table: google.cloud.bigquery.Table, - ordering: Optional[orderings.ExpressionOrdering], + ordering: Optional[orderings.TotalOrdering], ) -> ArrayValue: """ Replace the node with an equivalent one that references a tabel where the value has been materialized to. diff --git a/bigframes/core/compile/api.py b/bigframes/core/compile/api.py index 9fba3081ca..1f7d0a4507 100644 --- a/bigframes/core/compile/api.py +++ b/bigframes/core/compile/api.py @@ -46,7 +46,7 @@ def compile_ordered( def compile_raw( node: bigframes.core.nodes.BigFrameNode, -) -> Tuple[str, bigframes.core.ordering.ExpressionOrdering]: +) -> Tuple[str, bigframes.core.ordering.TotalOrdering]: """Compile node into sql that exposes all columns, including hidden ordering-only columns.""" ir = compiler.compile_ordered_ir(node) sql = ir.raw_sql() diff --git a/bigframes/core/compile/compiled.py b/bigframes/core/compile/compiled.py index 907c918efd..cc601744c1 100644 --- a/bigframes/core/compile/compiled.py +++ b/bigframes/core/compile/compiled.py @@ -36,9 +36,9 @@ from bigframes.core.ordering import ( ascending_over, encode_order_string, - ExpressionOrdering, IntegerEncoding, OrderingExpression, + TotalOrdering, ) import bigframes.core.schema as schemata import bigframes.core.sql @@ -194,7 +194,7 @@ def _aggregate_base( if by_column_ids: result = table.group_by(by_column_ids).aggregate(**stats) # Must have deterministic ordering, so order by the unique "by" column - ordering = ExpressionOrdering( + ordering = TotalOrdering( tuple([ascending_over(column_id) for column_id in by_column_ids]), total_ordering_columns=frozenset(by_column_ids), ) @@ -210,7 +210,7 @@ def _aggregate_base( # Ordering is irrelevant for single-row output, but set ordering id regardless # as other ops(join etc.) expect it. # TODO: Maybe can make completely empty - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple([]), total_ordering_columns=frozenset([]), ) @@ -279,7 +279,7 @@ def row_count(self) -> OrderedIR: return OrderedIR( ibis_table, (ibis_table["count"],), - ordering=ExpressionOrdering( + ordering=TotalOrdering( ordering_value_columns=(ascending_over("count"),), total_ordering_columns=frozenset(["count"]), ), @@ -519,7 +519,7 @@ def __init__( table: ibis_types.Table, columns: Sequence[ibis_types.Value], hidden_ordering_columns: Optional[Sequence[ibis_types.Value]] = None, - ordering: ExpressionOrdering = ExpressionOrdering(), + ordering: TotalOrdering = TotalOrdering(), predicates: Optional[Collection[ibis_types.BooleanValue]] = None, ): super().__init__(table, columns, predicates) @@ -598,10 +598,7 @@ def from_pandas( return cls( keys_memtable, columns=[keys_memtable[column].name(column) for column in pd_df.columns], - ordering=ExpressionOrdering( - ordering_value_columns=tuple([ascending_over(ORDER_ID_COLUMN)]), - total_ordering_columns=frozenset([ORDER_ID_COLUMN]), - ), + ordering=TotalOrdering.from_offset_col(ORDER_ID_COLUMN), hidden_ordering_columns=(keys_memtable[ORDER_ID_COLUMN],), ) @@ -760,7 +757,7 @@ def explode(self, column_ids: typing.Sequence[str]) -> OrderedIR: ], table_w_unnest[unnest_offset_id], ] - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple( [ *self._ordering.ordering_value_columns, @@ -1153,7 +1150,7 @@ def _bake_ordering(self) -> OrderedIR: self._ibis_bindings[expr.scalar_expression.id] ) - new_ordering = ExpressionOrdering( + new_ordering = TotalOrdering( tuple(new_exprs), self._ordering.integer_encoding, self._ordering.string_encoding, @@ -1176,7 +1173,7 @@ def _project_offsets(self) -> OrderedIR: ordering_mode="offset_col", order_col_name=ORDER_ID_COLUMN ) columns = [table[column_name] for column_name in self._column_names] - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple([ascending_over(ORDER_ID_COLUMN)]), total_ordering_columns=frozenset([ORDER_ID_COLUMN]), integer_encoding=IntegerEncoding(True, is_sequential=True), @@ -1300,7 +1297,7 @@ class Builder: def __init__( self, table: ibis_types.Table, - ordering: ExpressionOrdering, + ordering: TotalOrdering, columns: Collection[ibis_types.Value] = (), hidden_ordering_columns: Collection[ibis_types.Value] = (), predicates: Optional[Collection[ibis_types.BooleanValue]] = None, diff --git a/bigframes/core/compile/compiler.py b/bigframes/core/compile/compiler.py index 9272b8ad1b..ca9c479fff 100644 --- a/bigframes/core/compile/compiler.py +++ b/bigframes/core/compile/compiler.py @@ -195,7 +195,7 @@ def compile_read_table_ordered(node: nodes.ReadTableNode): ) else: integer_encoding = bf_ordering.IntegerEncoding() - ordering = bf_ordering.ExpressionOrdering( + ordering = bf_ordering.TotalOrdering( ordering_value_columns, integer_encoding=integer_encoding, total_ordering_columns=frozenset(node.total_order_cols), diff --git a/bigframes/core/compile/concat.py b/bigframes/core/compile/concat.py index 41a8f97821..35cf9ec5b4 100644 --- a/bigframes/core/compile/concat.py +++ b/bigframes/core/compile/concat.py @@ -21,9 +21,9 @@ import bigframes.core.compile.compiled as compiled from bigframes.core.ordering import ( ascending_over, - ExpressionOrdering, reencode_order_string, StringEncoding, + TotalOrdering, ) ORDER_ID_COLUMN = "bigframes_ordering_id" @@ -83,7 +83,7 @@ def concat_ordered( ) tables.append(table) combined_table = ibis.union(*tables) - ordering = ExpressionOrdering( + ordering = TotalOrdering( ordering_value_columns=tuple([ascending_over(ORDER_ID_COLUMN)]), total_ordering_columns=frozenset([ORDER_ID_COLUMN]), string_encoding=StringEncoding(True, prefix_size + max_encoding_size), diff --git a/bigframes/core/compile/default_ordering.py b/bigframes/core/compile/default_ordering.py index d8bdc0546b..7d7a41f742 100644 --- a/bigframes/core/compile/default_ordering.py +++ b/bigframes/core/compile/default_ordering.py @@ -82,7 +82,7 @@ def gen_default_ordering(table: ibis.table, use_double_hash: bool = True): itertools.chain(original_column_ids, order_values) ) - ordering = order.ExpressionOrdering( + ordering = order.TotalOrdering( ordering_value_columns=tuple( order.ascending_over(col.get_name()) for col in order_values ), diff --git a/bigframes/core/compile/single_column.py b/bigframes/core/compile/single_column.py index dbf25891bf..9b621c9c79 100644 --- a/bigframes/core/compile/single_column.py +++ b/bigframes/core/compile/single_column.py @@ -16,8 +16,6 @@ from __future__ import annotations -from typing import Mapping - import ibis import ibis.expr.datatypes as ibis_dtypes import ibis.expr.types as ibis_types @@ -84,7 +82,7 @@ def join_by_column_ordered( ) # Preserve ordering accross joins. - ordering = join_orderings( + ordering = orderings.join_orderings( left._ordering, right._ordering, l_mapping, @@ -173,33 +171,3 @@ def value_to_join_key(value: ibis_types.Value): if not value.type().is_string(): value = value.cast(ibis_dtypes.str) return value.fillna(ibis_types.literal("$NULL_SENTINEL$")) - - -def join_orderings( - left: orderings.ExpressionOrdering, - right: orderings.ExpressionOrdering, - left_id_mapping: Mapping[str, str], - right_id_mapping: Mapping[str, str], - left_order_dominates: bool = True, -) -> orderings.ExpressionOrdering: - left_ordering_refs = [ - ref.remap_names(left_id_mapping) for ref in left.all_ordering_columns - ] - right_ordering_refs = [ - ref.remap_names(right_id_mapping) for ref in right.all_ordering_columns - ] - if left_order_dominates: - joined_refs = [*left_ordering_refs, *right_ordering_refs] - else: - joined_refs = [*right_ordering_refs, *left_ordering_refs] - - left_total_order_cols = frozenset( - [left_id_mapping[id] for id in left.total_ordering_columns] - ) - right_total_order_cols = frozenset( - [right_id_mapping[id] for id in right.total_ordering_columns] - ) - return orderings.ExpressionOrdering( - ordering_value_columns=tuple(joined_refs), - total_ordering_columns=left_total_order_cols | right_total_order_cols, - ) diff --git a/bigframes/core/nodes.py b/bigframes/core/nodes.py index 65a1bd8084..a703cf1969 100644 --- a/bigframes/core/nodes.py +++ b/bigframes/core/nodes.py @@ -373,7 +373,7 @@ class CachedTableNode(BigFrameNode): table_id: str = field() physical_schema: Tuple[bq.SchemaField, ...] = field() - ordering: typing.Optional[orderings.ExpressionOrdering] = field() + ordering: typing.Optional[orderings.TotalOrdering] = field() def __post_init__(self): # enforce invariants diff --git a/bigframes/core/ordering.py b/bigframes/core/ordering.py index 1562592720..406ca52731 100644 --- a/bigframes/core/ordering.py +++ b/bigframes/core/ordering.py @@ -94,10 +94,42 @@ class IntegerEncoding: @dataclass(frozen=True) -class ExpressionOrdering: - """Immutable object that holds information about the ordering of rows in a ArrayValue object.""" +class RowOrdering: + """Immutable object that holds information about the ordering of rows in a ArrayValue object. May not be unambiguous.""" ordering_value_columns: typing.Tuple[OrderingExpression, ...] = () + + @property + def all_ordering_columns(self) -> Sequence[OrderingExpression]: + return list(self.ordering_value_columns) + + @property + def referenced_columns(self) -> Set[str]: + return set( + col + for part in self.ordering_value_columns + for col in part.scalar_expression.unbound_variables + ) + + def with_reverse(self) -> RowOrdering: + """Reverses the ordering.""" + return RowOrdering( + tuple([col.with_reverse() for col in self.ordering_value_columns]), + ) + + def with_column_remap(self, mapping: typing.Mapping[str, str]) -> RowOrdering: + new_value_columns = [ + col.remap_names(mapping) for col in self.all_ordering_columns + ] + return TotalOrdering( + tuple(new_value_columns), + ) + + +@dataclass(frozen=True) +class TotalOrdering(RowOrdering): + """Immutable object that holds information about the ordering of rows in a ArrayValue object. Guaranteed to be unambiguous.""" + integer_encoding: IntegerEncoding = IntegerEncoding(False) string_encoding: StringEncoding = StringEncoding(False) # A table has a total ordering defined by the identities of a set of 1 or more columns. @@ -106,8 +138,8 @@ class ExpressionOrdering: total_ordering_columns: frozenset[str] = field(default_factory=frozenset) @classmethod - def from_offset_col(cls, col: str) -> ExpressionOrdering: - return ExpressionOrdering( + def from_offset_col(cls, col: str) -> TotalOrdering: + return TotalOrdering( (ascending_over(col),), integer_encoding=IntegerEncoding(True, is_sequential=True), total_ordering_columns=frozenset({col}), @@ -119,7 +151,7 @@ def with_non_sequential(self): This is useful when filtering, but not sorting, an expression. """ if self.integer_encoding.is_sequential: - return ExpressionOrdering( + return TotalOrdering( self.ordering_value_columns, integer_encoding=IntegerEncoding( self.integer_encoding.is_encoded, is_sequential=False @@ -132,7 +164,7 @@ def with_non_sequential(self): def with_ordering_columns( self, ordering_value_columns: Sequence[OrderingExpression] = (), - ) -> ExpressionOrdering: + ) -> TotalOrdering: """Creates a new ordering that reorders by the given columns. Args: @@ -147,7 +179,7 @@ def with_ordering_columns( new_ordering = self._truncate_ordering( (*ordering_value_columns, *self.ordering_value_columns) ) - return ExpressionOrdering( + return TotalOrdering( new_ordering, total_ordering_columns=self.total_ordering_columns, ) @@ -173,7 +205,7 @@ def _truncate_ordering( def with_reverse(self): """Reverses the ordering.""" - return ExpressionOrdering( + return TotalOrdering( tuple([col.with_reverse() for col in self.ordering_value_columns]), total_ordering_columns=self.total_ordering_columns, ) @@ -185,7 +217,7 @@ def with_column_remap(self, mapping: typing.Mapping[str, str]): new_total_order = frozenset( mapping.get(col_id, col_id) for col_id in self.total_ordering_columns ) - return ExpressionOrdering( + return TotalOrdering( tuple(new_value_columns), integer_encoding=self.integer_encoding, string_encoding=self.string_encoding, @@ -211,18 +243,6 @@ def is_string_encoded(self) -> bool: def is_sequential(self) -> bool: return self.integer_encoding.is_encoded and self.integer_encoding.is_sequential - @property - def all_ordering_columns(self) -> Sequence[OrderingExpression]: - return list(self.ordering_value_columns) - - @property - def referenced_columns(self) -> Set[str]: - return set( - col - for part in self.ordering_value_columns - for col in part.scalar_expression.unbound_variables - ) - def encode_order_string( order_id: ibis_types.IntegerColumn, length: int = DEFAULT_ORDERING_ID_LENGTH @@ -257,3 +277,58 @@ def descending_over(id: str, nulls_last: bool = True) -> OrderingExpression: return OrderingExpression( expression.free_var(id), direction=OrderingDirection.DESC, na_last=nulls_last ) + + +@typing.overload +def join_orderings( + left: TotalOrdering, + right: TotalOrdering, + left_id_mapping: Mapping[str, str], + right_id_mapping: Mapping[str, str], + left_order_dominates: bool = True, +) -> TotalOrdering: + ... + + +@typing.overload +def join_orderings( + left: RowOrdering, + right: RowOrdering, + left_id_mapping: Mapping[str, str], + right_id_mapping: Mapping[str, str], + left_order_dominates: bool = True, +) -> RowOrdering: + ... + + +def join_orderings( + left: RowOrdering, + right: RowOrdering, + left_id_mapping: Mapping[str, str], + right_id_mapping: Mapping[str, str], + left_order_dominates: bool = True, +) -> RowOrdering: + left_ordering_refs = [ + ref.remap_names(left_id_mapping) for ref in left.all_ordering_columns + ] + right_ordering_refs = [ + ref.remap_names(right_id_mapping) for ref in right.all_ordering_columns + ] + if left_order_dominates: + joined_refs = [*left_ordering_refs, *right_ordering_refs] + else: + joined_refs = [*right_ordering_refs, *left_ordering_refs] + + if isinstance(left, TotalOrdering) and isinstance(right, TotalOrdering): + left_total_order_cols = frozenset( + [left_id_mapping[id] for id in left.total_ordering_columns] + ) + right_total_order_cols = frozenset( + [right_id_mapping[id] for id in right.total_ordering_columns] + ) + return TotalOrdering( + ordering_value_columns=tuple(joined_refs), + total_ordering_columns=left_total_order_cols | right_total_order_cols, + ) + else: + return RowOrdering(tuple(joined_refs)) diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index 867bdedf1c..02d30cdbd2 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -1910,7 +1910,7 @@ def _cache_with_offsets(self, array_value: core.ArrayValue): ) cached_replacement = array_value.as_cached( cache_table=self.bqclient.get_table(tmp_table), - ordering=order.ExpressionOrdering.from_offset_col(offset_column), + ordering=order.TotalOrdering.from_offset_col(offset_column), ).node self._cached_executions[array_value.node] = cached_replacement From 74170dabd323f1b08ad76241e37ff9f2a5b67ab5 Mon Sep 17 00:00:00 2001 From: Garrett Wu <6505921+GarrettWu@users.noreply.github.com> Date: Wed, 3 Jul 2024 18:05:52 -0700 Subject: [PATCH 5/8] docs: remove session and connection in llm notebook (#821) --- .../generative_ai/large_language_models.ipynb | 106 +++++++----------- 1 file changed, 41 insertions(+), 65 deletions(-) diff --git a/notebooks/generative_ai/large_language_models.ipynb b/notebooks/generative_ai/large_language_models.ipynb index 08ef52b544..744706cab8 100644 --- a/notebooks/generative_ai/large_language_models.ipynb +++ b/notebooks/generative_ai/large_language_models.ipynb @@ -16,8 +16,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Prerequisites\n", - "Create session and define a BQ connection which we already created and allowlisted. " + "## Define the model" ] }, { @@ -29,33 +28,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/usr/local/google/home/garrettwu/src/bigframes/bigframes/session/__init__.py:1762: UserWarning: No explicit location is set, so using location US for the session.\n", - " return Session(context)\n" + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/ml/llm.py:589: DefaultLocationWarning: No explicit location is set, so using location US for the session.\n", + " self.session = session or bpd.get_global_session()\n" ] - } - ], - "source": [ - "session = bigframes.pandas.get_global_session()\n", - "connection = f\"{session.bqclient.project}.us.bigframes-default-connection\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Define the model" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ + }, { "data": { "text/html": [ - "Query job 12bcd690-ca99-4001-bf26-032f50e77d62 is DONE. 0 Bytes processed. Open Job" + "Query job 675a6c8a-213b-496c-9f77-b87bf7cfa5e0 is DONE. 0 Bytes processed. Open Job" ], "text/plain": [ "" @@ -66,7 +46,7 @@ } ], "source": [ - "model = GeminiTextGenerator(session=session, connection_name=connection)" + "model = GeminiTextGenerator()" ] }, { @@ -81,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -102,13 +82,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "Query job f8fe31c6-7d8a-4919-9492-8304a0083cca is DONE. 0 Bytes processed. Open Job" + "Query job 7967df2b-9f0f-45c8-a363-15f65891c3bf is DONE. 0 Bytes processed. Open Job" ], "text/plain": [ "" @@ -118,21 +98,17 @@ "output_type": "display_data" }, { - "data": { - "text/html": [ - "Query job 28bab71f-e218-4d92-9a50-dab41bb0c71f is DONE. 24 Bytes processed. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/garrettwu/src/bigframes/bigframes/core/__init__.py:108: PreviewWarning: Interpreting JSON column(s) as StringDtype. This behavior may change in future versions.\n", + " warnings.warn(\n" + ] }, { "data": { "text/html": [ - "Query job 01d66b61-459f-474e-9f66-d519f9c2f23d is DONE. 6 Bytes processed. Open Job" + "Query job 9a1f57cd-98e1-4eac-a1b3-8f88d61971cd is DONE. 6 Bytes processed. Open Job" ], "text/plain": [ "" @@ -144,7 +120,7 @@ { "data": { "text/html": [ - "Query job af606ca7-4bcf-4bd1-95fd-c516542b5a4f is DONE. 5.3 kB processed. Open Job" + "Query job 2a94a2cf-7d4c-4009-a798-d7a5d6d4049d is DONE. 8.5 kB processed. Open Job" ], "text/plain": [ "" @@ -183,28 +159,28 @@ " \n", " \n", " 0\n", - " **BigQuery**\n", + " ## BigQuery: A Serverless Data Warehouse\n", "\n", - "**Definition:**\n", - "\n", - "BigQuery is a s...\n", - " null\n", + "BigQ...\n", + " [{\"category\":1,\"probability\":1,\"probability_sc...\n", " \n", " What is BigQuery?\n", " \n", " \n", " 1\n", - " **BigQuery Machine Learning (BQML)**\n", + " ## BigQuery Machine Learning (BQML)\n", "\n", - "BQML is ...\n", - " null\n", + "BQML is a...\n", + " [{\"category\":1,\"probability\":1,\"probability_sc...\n", " \n", " What is BQML?\n", " \n", " \n", " 2\n", - " BigQuery DataFrame is a Python DataFrame imple...\n", - " null\n", + " ## What is BigQuery DataFrame?\n", + "\n", + "**BigQuery Dat...\n", + " [{\"category\":1,\"probability\":1,\"probability_sc...\n", " \n", " What is BigQuery DataFrame?\n", " \n", @@ -214,20 +190,20 @@ ], "text/plain": [ " ml_generate_text_llm_result \\\n", - "0 **BigQuery**\n", + "0 ## BigQuery: A Serverless Data Warehouse\n", "\n", - "**Definition:**\n", + "BigQ... \n", + "1 ## BigQuery Machine Learning (BQML)\n", "\n", - "BigQuery is a s... \n", - "1 **BigQuery Machine Learning (BQML)**\n", + "BQML is a... \n", + "2 ## What is BigQuery DataFrame?\n", "\n", - "BQML is ... \n", - "2 BigQuery DataFrame is a Python DataFrame imple... \n", + "**BigQuery Dat... \n", "\n", - " ml_generate_text_rai_result ml_generate_text_status \\\n", - "0 null \n", - "1 null \n", - "2 null \n", + " ml_generate_text_rai_result ml_generate_text_status \\\n", + "0 [{\"category\":1,\"probability\":1,\"probability_sc... \n", + "1 [{\"category\":1,\"probability\":1,\"probability_sc... \n", + "2 [{\"category\":1,\"probability\":1,\"probability_sc... \n", "\n", " prompt \n", "0 What is BigQuery? \n", @@ -235,7 +211,7 @@ "2 What is BigQuery DataFrame? " ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -255,16 +231,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'**BigQuery**\\n\\n**Definition:**\\n\\nBigQuery is a serverless, highly scalable, cloud-based data warehouse and analytics platform offered by Google Cloud.\\n\\n**Key Features:**\\n\\n* **Massive Scalability:** Can handle large datasets (petabytes or more) with fast query execution.\\n* **Elastic:** Automatically scales compute resources based on workload requirements.\\n* **Serverless:** Users do not need to manage infrastructure or provision resources.\\n* **Flexible Data Loading:** Supports a wide range of data sources, including files, databases, and streaming data.\\n* **SQL-Based Querying:** Uses standard SQL syntax for querying and analyzing data.\\n* **Machine Learning Integration:** Provides built-in machine learning capabilities for predictive analytics and data exploration.\\n* **Real-Time Analysis:** Supports streaming data analysis and interactive dashboards.\\n* **Collaboration and Sharing:** Allows multiple users to access and analyze data in a collaborative environment.\\n* **Cost-Effective:** Pay-as-you-go pricing based on data scanned and compute resources used.\\n\\n**Applications:**\\n\\n* Data warehousing and analytics\\n* Business intelligence and reporting\\n* Data science and machine learning\\n* Data exploration and visualization\\n* Marketing analytics\\n* Fraud detection and risk management\\n\\n**Benefits:**\\n\\n* Rapid data analysis on large datasets\\n* Reduced infrastructure management overhead\\n* Increased agility and flexibility\\n* Enhanced collaboration and data sharing\\n* Cost-effective data storage and analytics'" + "\"## BigQuery: A Serverless Data Warehouse\\n\\nBigQuery is a serverless, cloud-based data warehouse that enables scalable analysis of large datasets. It's a popular choice for businesses of all sizes due to its ability to handle petabytes of data and run complex queries quickly and efficiently. Let's delve into its key features:\\n\\n**Serverless Architecture:** BigQuery eliminates the need for server management, allowing you to focus on analyzing data. Google manages the infrastructure, scaling resources up or down automatically based on your needs.\\n\\n**Scalability:** BigQuery can handle massive datasets, scaling seamlessly as your data volume grows. It automatically distributes queries across its infrastructure, ensuring fast and efficient processing.\\n\\n**SQL-like Querying:** BigQuery uses a familiar SQL-like syntax, making it easy for data analysts and developers to learn and use. This allows them to leverage their existing SQL knowledge for data exploration and analysis.\\n\\n**Cost-Effectiveness:** BigQuery offers a pay-as-you-go pricing model, meaning you only pay for the resources you use. This makes it a cost-effective solution for businesses with varying data processing needs.\\n\\n**Integration with Google Cloud:** BigQuery integrates seamlessly with other Google Cloud services like Cloud Storage, Dataflow, and Machine Learning, enabling a comprehensive data processing and analysis workflow within the Google Cloud ecosystem.\\n\\n**Security and Reliability:** BigQuery offers robust security features and high availability, ensuring data protection and reliable access.\\n\\n**Use Cases:** BigQuery finds applications in various scenarios, including:\\n\\n* **Data Warehousing:** Store and analyze large amounts of structured and semi-structured data.\\n* **Business Intelligence:** Generate insights from data for informed decision-making.\\n* **Data Analytics:** Perform complex data analysis and extract valuable patterns.\\n* **Machine Learning:** Train and deploy machine learning models on large datasets.\\n\\n**Getting Started:** To get started with BigQuery, you can create a free trial account on Google Cloud Platform and explore its features. Numerous tutorials and documentation are available to help you learn and use BigQuery effectively.\\n\\n## Additional Resources:\\n\\n* **BigQuery Documentation:** https://cloud.google.com/bigquery/docs/\\n* **BigQuery Quickstart:** https://cloud.google.com/bigquery/docs/quickstarts/quickstart-console\\n* **BigQuery Pricing:** https://cloud.google.com/bigquery/pricing\\n\\nFeel free to ask if you have any further questions about BigQuery!\"" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } From 067ff173f0abfcf5bf06d3fbdb6d12e0fa5283c3 Mon Sep 17 00:00:00 2001 From: Shobhit Singh Date: Mon, 8 Jul 2024 10:47:50 -0700 Subject: [PATCH 6/8] docs: remove the experimental flask icon from the public docs (#820) --- docs/templates/toc.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/templates/toc.yml b/docs/templates/toc.yml index aee3058ecc..95bded9a60 100644 --- a/docs/templates/toc.yml +++ b/docs/templates/toc.yml @@ -204,4 +204,3 @@ uid: bigframes.bigquery name: bigframes.bigquery name: BigQuery DataFrames - status: beta From 7cae3f41cee2600bc901e084974203931b065ee2 Mon Sep 17 00:00:00 2001 From: Chelsea Lin <124939984+chelsea-lin@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:44:18 -0700 Subject: [PATCH 7/8] chore: remove an unused parameter in _rows_to_dataframe (#813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery-dataframes/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [X] Ensure the tests and linter pass - [X] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes # 🦕 --- bigframes/core/blocks.py | 7 ++----- bigframes/session/__init__.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/bigframes/core/blocks.py b/bigframes/core/blocks.py index 25acfe2f42..fef91f88dc 100644 --- a/bigframes/core/blocks.py +++ b/bigframes/core/blocks.py @@ -438,9 +438,7 @@ def reorder_levels(self, ids: typing.Sequence[str]): def _to_dataframe(self, result) -> pd.DataFrame: """Convert BigQuery data to pandas DataFrame with specific dtypes.""" - dtypes = dict(zip(self.index_columns, self.index.dtypes)) - dtypes.update(zip(self.value_columns, self.dtypes)) - result_dataframe = self.session._rows_to_dataframe(result, dtypes) + result_dataframe = self.session._rows_to_dataframe(result) # Runs strict validations to ensure internal type predictions and ibis are completely in sync # Do not execute these validations outside of testing suite. if "PYTEST_CURRENT_TEST" in os.environ: @@ -2582,7 +2580,6 @@ def to_pandas(self, *, ordered: Optional[bool] = None) -> pd.Index: ) # Project down to only the index column. So the query can be cached to visualize other data. index_columns = list(self._block.index_columns) - dtypes = dict(zip(index_columns, self.dtypes)) expr = self._expr.select_columns(index_columns) results, _ = self.session._execute( expr, @@ -2590,7 +2587,7 @@ def to_pandas(self, *, ordered: Optional[bool] = None) -> pd.Index: if (ordered is not None) else self.session._strictly_ordered, ) - df = expr.session._rows_to_dataframe(results, dtypes) + df = expr.session._rows_to_dataframe(results) df = df.set_index(index_columns) index = df.index index.names = list(self._block._index_labels) # type:ignore diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index 02d30cdbd2..0f7953d3d4 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -2038,7 +2038,7 @@ def _get_table_size(self, destination_table): return table.num_bytes def _rows_to_dataframe( - self, row_iterator: bigquery.table.RowIterator, dtypes: Dict + self, row_iterator: bigquery.table.RowIterator ) -> pandas.DataFrame: # Can ignore inferred datatype until dtype emulation breaks 1:1 mapping between BQ types and bigframes types dtypes_from_bq = bigframes.dtypes.bf_type_from_type_kind(row_iterator.schema) From ee2b660f15e1ab29de6e8abfba093c7839b14918 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 18:28:02 -0700 Subject: [PATCH 8/8] chore(main): release 1.11.1 (#822) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ bigframes/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b27da8d301..8249515719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/bigframes/#history +## [1.11.1](https://github.com/googleapis/python-bigquery-dataframes/compare/v1.11.0...v1.11.1) (2024-07-08) + + +### Documentation + +* Remove session and connection in llm notebook ([#821](https://github.com/googleapis/python-bigquery-dataframes/issues/821)) ([74170da](https://github.com/googleapis/python-bigquery-dataframes/commit/74170dabd323f1b08ad76241e37ff9f2a5b67ab5)) +* Remove the experimental flask icon from the public docs ([#820](https://github.com/googleapis/python-bigquery-dataframes/issues/820)) ([067ff17](https://github.com/googleapis/python-bigquery-dataframes/commit/067ff173f0abfcf5bf06d3fbdb6d12e0fa5283c3)) + ## [1.11.0](https://github.com/googleapis/python-bigquery-dataframes/compare/v1.10.0...v1.11.0) (2024-07-01) diff --git a/bigframes/version.py b/bigframes/version.py index 75f3ffb361..1186811c97 100644 --- a/bigframes/version.py +++ b/bigframes/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.11.0" +__version__ = "1.11.1" pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy