Content-Length: 507642 | pFad | http://github.com/apache/airflow/pull/23165/commits/44343a90f4e61c2e771922049f71b9f55a48fbc9

57 Bigquery assets by wojsamjan · Pull Request #23165 · apache/airflow · GitHub
Skip to content

Bigquery assets #23165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 30, 2022
Merged

Bigquery assets #23165

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
44343a9
Fix tests
Apr 19, 2022
209c074
Add BigQuery links
Apr 19, 2022
c863b18
Fix types
Apr 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Fix tests
  • Loading branch information
Wojciech Januszek committed Apr 26, 2022
commit 44343a90f4e61c2e771922049f71b9f55a48fbc9
10 changes: 0 additions & 10 deletions tests/providers/google/cloud/hooks/test_bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -937,16 +937,6 @@ def test_insert_job(self, mock_client, mock_query_job, nowait):
def test_dbapi_get_uri(self):
assert self.hook.get_uri().startswith('bigquery://')

def test_dbapi_get_sqlalchemy_engine_failed(self):
with pytest.raises(
AirflowException,
match="For now, we only support instantiating SQLAlchemy engine by"
" using ADC"
", extra__google_cloud_platform__key_path"
"and extra__google_cloud_platform__keyfile_dict",
):
self.hook.get_sqlalchemy_engine()


class TestBigQueryTableSplitter(unittest.TestCase):
def test_internal_need_default_project(self):
Expand Down
30 changes: 15 additions & 15 deletions tests/providers/google/cloud/operators/test_bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def test_execute(self, mock_hook):
task_id=TASK_ID, dataset_id=TEST_DATASET, project_id=TEST_GCP_PROJECT_ID, table_id=TEST_TABLE_ID
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.create_empty_table.assert_called_once_with(
dataset_id=TEST_DATASET,
project_id=TEST_GCP_PROJECT_ID,
Expand All @@ -108,7 +108,7 @@ def test_create_view(self, mock_hook):
view=VIEW_DEFINITION,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.create_empty_table.assert_called_once_with(
dataset_id=TEST_DATASET,
project_id=TEST_GCP_PROJECT_ID,
Expand All @@ -134,7 +134,7 @@ def test_create_materialized_view(self, mock_hook):
materialized_view=MATERIALIZED_VIEW_DEFINITION,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.create_empty_table.assert_called_once_with(
dataset_id=TEST_DATASET,
project_id=TEST_GCP_PROJECT_ID,
Expand Down Expand Up @@ -170,7 +170,7 @@ def test_create_clustered_empty_table(self, mock_hook):
cluster_fields=cluster_fields,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.create_empty_table.assert_called_once_with(
dataset_id=TEST_DATASET,
project_id=TEST_GCP_PROJECT_ID,
Expand Down Expand Up @@ -200,7 +200,7 @@ def test_execute(self, mock_hook):
autodetect=True,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.create_external_table.assert_called_once_with(
external_project_dataset_table=f'{TEST_DATASET}.{TEST_TABLE_ID}',
schema_fields=[],
Expand Down Expand Up @@ -246,7 +246,7 @@ def test_execute(self, mock_hook):
location=TEST_DATASET_LOCATION,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.create_empty_dataset.assert_called_once_with(
dataset_id=TEST_DATASET,
project_id=TEST_GCP_PROJECT_ID,
Expand All @@ -263,7 +263,7 @@ def test_execute(self, mock_hook):
task_id=TASK_ID, dataset_id=TEST_DATASET, project_id=TEST_GCP_PROJECT_ID
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.get_dataset.assert_called_once_with(
dataset_id=TEST_DATASET, project_id=TEST_GCP_PROJECT_ID
)
Expand All @@ -281,7 +281,7 @@ def test_execute(self, mock_hook):
project_id=TEST_GCP_PROJECT_ID,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.update_table.assert_called_once_with(
table_resource=table_resource,
fields=None,
Expand Down Expand Up @@ -310,7 +310,7 @@ def test_execute(self, mock_hook):
table_id=TEST_TABLE_ID,
project_id=TEST_GCP_PROJECT_ID,
)
operator.execute(None)
operator.execute(context=MagicMock())

mock_hook.return_value.update_table_schema.assert_called_once_with(
schema_fields_updates=schema_field_updates,
Expand Down Expand Up @@ -349,7 +349,7 @@ def test_execute(self, mock_hook):
project_id=TEST_GCP_PROJECT_ID,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.update_dataset.assert_called_once_with(
dataset_resource=dataset_resource,
dataset_id=TEST_DATASET,
Expand Down Expand Up @@ -806,7 +806,7 @@ def test_execute(self, mock_hook):
project_id=TEST_GCP_PROJECT_ID,
)

operator.execute(None)
operator.execute(context=MagicMock())
mock_hook.return_value.run_table_upsert.assert_called_once_with(
dataset_id=TEST_DATASET, project_id=TEST_GCP_PROJECT_ID, table_resource=TEST_TABLE_RESOURCES
)
Expand Down Expand Up @@ -836,7 +836,7 @@ def test_execute_success(self, mock_hook, mock_md5):
job_id=job_id,
project_id=TEST_GCP_PROJECT_ID,
)
result = op.execute({})
result = op.execute(context=MagicMock())

mock_hook.return_value.insert_job.assert_called_once_with(
configuration=configuration,
Expand Down Expand Up @@ -873,7 +873,7 @@ def test_on_kill(self, mock_hook, mock_md5):
project_id=TEST_GCP_PROJECT_ID,
cancel_on_kill=False,
)
op.execute({})
op.execute(context=MagicMock())

op.on_kill()
mock_hook.return_value.cancel_job.assert_not_called()
Expand Down Expand Up @@ -944,7 +944,7 @@ def test_execute_reattach(self, mock_hook, mock_md5):
project_id=TEST_GCP_PROJECT_ID,
reattach_states={"PENDING"},
)
result = op.execute({})
result = op.execute(context=MagicMock())

mock_hook.return_value.get_job.assert_called_once_with(
location=TEST_DATASET_LOCATION,
Expand Down Expand Up @@ -989,7 +989,7 @@ def test_execute_force_rerun(self, mock_hook, mock_uuid, mock_md5):
project_id=TEST_GCP_PROJECT_ID,
force_rerun=True,
)
result = op.execute({})
result = op.execute(context=MagicMock())

mock_hook.return_value.insert_job.assert_called_once_with(
configuration=configuration,
Expand Down
2 changes: 1 addition & 1 deletion tests/providers/google/cloud/transfers/test_bigquery_to_bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def test_execute(self, mock_hook):
encryption_configuration=encryption_configuration,
)

operator.execute(None)
operator.execute(context=mock.MagicMock())
mock_hook.return_value.run_copy.assert_called_once_with(
source_project_dataset_tables=source_project_dataset_tables,
destination_project_dataset_table=destination_project_dataset_table,
Expand Down
2 changes: 1 addition & 1 deletion tests/providers/google/cloud/transfers/test_bigquery_to_gcs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def test_execute(self, mock_hook):
labels=labels,
)

operator.execute(None)
operator.execute(context=mock.MagicMock())

mock_hook.return_value.run_extract.assert_called_once_with(
source_project_dataset_table=source_project_dataset_table,
Expand Down
2 changes: 1 addition & 1 deletion tests/providers/google/cloud/transfers/test_bigquery_to_mssql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def test_execute_good_request_to_bq(self, mock_hook):
replace=False,
)

operator.execute(None)
operator.execute(context=mock.MagicMock())
# fmt: off
mock_hook.return_value.list_rows.assert_called_once_with(
dataset_id=TEST_DATASET,
Expand Down








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: http://github.com/apache/airflow/pull/23165/commits/44343a90f4e61c2e771922049f71b9f55a48fbc9

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy