Content-Length: 788067 | pFad | https://github.com/apache/airflow/commit/3a539ff6631109dc58514339ac60672f031c7054

A4 Life Science assets & system tests migration (AIP-47) (#25548) · apache/airflow@3a539ff · GitHub
Skip to content

Commit 3a539ff

Browse files
bkossakowskabkossakowska
authored
Life Science assets & system tests migration (AIP-47) (#25548)
1 parent 5066844 commit 3a539ff

File tree

9 files changed

+126
-16
lines changed

9 files changed

+126
-16
lines changed

‎airflow/providers/google/cloud/links/life_sciences.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
from typing import TYPE_CHECKING
19+
20+
from airflow.providers.google.cloud.links.base import BaseGoogleLink
21+
22+
if TYPE_CHECKING:
23+
from airflow.utils.context import Context
24+
25+
BASE_LINK = "https://console.cloud.google.com/lifesciences"
26+
LIFESCIENCES_LIST_LINK = BASE_LINK + "/pipelines?project={project_id}"
27+
28+
29+
class LifeSciencesLink(BaseGoogleLink):
30+
"""Helper class for constructing Life Sciences List link"""
31+
32+
name = "Life Sciences"
33+
key = "lifesciences_key"
34+
format_str = LIFESCIENCES_LIST_LINK
35+
36+
@staticmethod
37+
def persist(
38+
context: "Context",
39+
task_instance,
40+
project_id: str,
41+
):
42+
task_instance.xcom_push(
43+
context=context,
44+
key=LifeSciencesLink.key,
45+
value={
46+
"project_id": project_id,
47+
},
48+
)

‎airflow/providers/google/cloud/operators/life_sciences.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from airflow.exceptions import AirflowException
2323
from airflow.models import BaseOperator
2424
from airflow.providers.google.cloud.hooks.life_sciences import LifeSciencesHook
25+
from airflow.providers.google.cloud.links.life_sciences import LifeSciencesLink
2526

2627
if TYPE_CHECKING:
2728
from airflow.utils.context import Context
@@ -57,6 +58,7 @@ class LifeSciencesRunPipelineOperator(BaseOperator):
5758
"api_version",
5859
"impersonation_chain",
5960
)
61+
operator_extra_links = (LifeSciencesLink(),)
6062

6163
def __init__(
6264
self,
@@ -90,5 +92,11 @@ def execute(self, context: 'Context') -> dict:
9092
api_version=self.api_version,
9193
impersonation_chain=self.impersonation_chain,
9294
)
93-
95+
project_id = self.project_id or hook.project_id
96+
if project_id:
97+
LifeSciencesLink.persist(
98+
context=context,
99+
task_instance=self,
100+
project_id=project_id,
101+
)
94102
return hook.run_pipeline(body=self.body, location=self.location, project_id=self.project_id)

‎airflow/providers/google/provider.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,6 +1014,7 @@ extra-links:
10141014
- airflow.providers.google.cloud.links.cloud_build.CloudBuildListLink
10151015
- airflow.providers.google.cloud.links.cloud_build.CloudBuildTriggersListLink
10161016
- airflow.providers.google.cloud.links.cloud_build.CloudBuildTriggerDetailsLink
1017+
- airflow.providers.google.cloud.links.life_sciences.LifeSciencesLink
10171018
- airflow.providers.google.common.links.storage.StorageLink
10181019
- airflow.providers.google.common.links.storage.FileDetailsLink
10191020

‎tests/providers/google/cloud/operators/test_life_sciences.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ def test_executes(self, mock_hook):
4242
operator = LifeSciencesRunPipelineOperator(
4343
task_id='task-id', body=TEST_BODY, location=TEST_LOCATION, project_id=TEST_PROJECT_ID
4444
)
45-
result = operator.execute(None)
45+
context = mock.MagicMock()
46+
result = operator.execute(context=context)
47+
4648
assert result == TEST_OPERATION
4749

4850
@mock.patch("airflow.providers.google.cloud.operators.life_sciences.LifeSciencesHook")
@@ -54,5 +56,6 @@ def test_executes_without_project_id(self, mock_hook):
5456
body=TEST_BODY,
5557
location=TEST_LOCATION,
5658
)
57-
result = operator.execute(None)
59+
context = mock.MagicMock()
60+
result = operator.execute(context=context)
5861
assert result == TEST_OPERATION

‎tests/providers/google/cloud/transfers/test_gdrive_to_gcs_system.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import pytest
1919

2020
from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY
21-
from tests.system.providers.google.cloud.life_sciences.example_life_sciences import BUCKET
21+
from tests.system.providers.google.cloud.life_sciences.example_life_sciences import BUCKET_NAME
2222
from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
2323

2424

@@ -28,13 +28,13 @@ class GoogleDriveToGCSExampleDagsSystemTest(GoogleSystemTest):
2828
@provide_gcp_context(GCP_GCS_KEY)
2929
def setUp(self):
3030
super().setUp()
31-
self.create_gcs_bucket(BUCKET)
31+
self.create_gcs_bucket(BUCKET_NAME)
3232

3333
@provide_gcp_context(GCP_GCS_KEY)
3434
def test_run_example_dag_function(self):
3535
self.run_dag('example_gdrive_to_gcs', CLOUD_DAG_FOLDER)
3636

3737
@provide_gcp_context(GCP_GCS_KEY)
3838
def tearDown(self):
39-
self.delete_gcs_bucket(BUCKET)
39+
self.delete_gcs_bucket(BUCKET_NAME)
4040
super().tearDown()

‎tests/providers/google/cloud/utils/gcp_authenticator.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@
4848
GCP_GCS_TRANSFER_KEY = 'gcp_gcs_transfer.json'
4949
GCP_GKE_KEY = "gcp_gke.json"
5050
GCP_KMS_KEY = "gcp_kms.json"
51-
GCP_LIFE_SCIENCES_KEY = 'gcp_life_sciences.json'
5251
GCP_MEMORYSTORE = 'gcp_memorystore.json'
5352
GCP_PUBSUB_KEY = "gcp_pubsub.json"
5453
GCP_SECRET_MANAGER_KEY = 'gcp_secret_manager.json'

‎tests/system/providers/google/cloud/life_sciences/example_life_sciences.py

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,26 @@
1818

1919
import os
2020
from datetime import datetime
21+
from pathlib import Path
2122

2223
from airflow import models
24+
from airflow.models.baseoperator import chain
25+
from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
2326
from airflow.providers.google.cloud.operators.life_sciences import LifeSciencesRunPipelineOperator
27+
from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
28+
from airflow.utils.trigger_rule import TriggerRule
2429

2530
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
26-
DAG_ID = "example_gcp_life_sciences"
31+
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
32+
DAG_ID = "example_life_sciences"
2733

28-
PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project-id")
29-
BUCKET = os.environ.get("GCP_GCS_LIFE_SCIENCES_BUCKET", "INVALID BUCKET NAME")
30-
FILENAME = os.environ.get("GCP_GCS_LIFE_SCIENCES_FILENAME", 'input.in')
31-
LOCATION = os.environ.get("GCP_LIFE_SCIENCES_LOCATION", 'us-central1')
34+
BUCKET_NAME = f"bucket_{DAG_ID}-{ENV_ID}"
3235

36+
FILE_NAME = "file"
37+
LOCATION = "us-central1"
38+
39+
CURRENT_FOLDER = Path(__file__).parent
40+
FILE_LOCAL_PATH = str(Path(CURRENT_FOLDER) / "resources" / FILE_NAME)
3341

3442
# [START howto_configure_simple_action_pipeline]
3543
SIMPLE_ACTION_PIPELINE = {
@@ -53,16 +61,16 @@
5361
"actions": [
5462
{
5563
"imageUri": "google/cloud-sdk",
56-
"commands": ["gsutil", "cp", f"gs://{BUCKET}/{FILENAME}", "/tmp"],
64+
"commands": ["gsutil", "cp", f"gs://{BUCKET_NAME}/{FILE_NAME}", "/tmp"],
5765
},
5866
{"imageUri": "bash", "commands": ["-c", "echo Hello, world"]},
5967
{
6068
"imageUri": "google/cloud-sdk",
6169
"commands": [
6270
"gsutil",
6371
"cp",
64-
f"gs://{BUCKET}/{FILENAME}",
65-
f"gs://{BUCKET}/output.in",
72+
f"gs://{BUCKET_NAME}/{FILE_NAME}",
73+
f"gs://{BUCKET_NAME}/output.in",
6674
],
6775
},
6876
],
@@ -83,6 +91,14 @@
8391
catchup=False,
8492
tags=['example'],
8593
) as dag:
94+
create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=BUCKET_NAME)
95+
96+
upload_file = LocalFilesystemToGCSOperator(
97+
task_id="upload_file",
98+
src=FILE_LOCAL_PATH,
99+
dst=FILE_NAME,
100+
bucket=BUCKET_NAME,
101+
)
86102

87103
# [START howto_run_pipeline]
88104
simple_life_science_action_pipeline = LifeSciencesRunPipelineOperator(
@@ -97,7 +113,26 @@
97113
task_id='multi-action-pipeline', body=MULTI_ACTION_PIPELINE, project_id=PROJECT_ID, location=LOCATION
98114
)
99115

100-
simple_life_science_action_pipeline >> multiple_life_science_action_pipeline
116+
delete_bucket = GCSDeleteBucketOperator(
117+
task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE
118+
)
119+
120+
chain(
121+
# TEST SETUP
122+
create_bucket,
123+
upload_file,
124+
# TEST BODY
125+
simple_life_science_action_pipeline,
126+
multiple_life_science_action_pipeline,
127+
# TEST TEARDOWN
128+
delete_bucket,
129+
)
130+
131+
from tests.system.utils.watcher import watcher
132+
133+
# This test needs watcher in order to properly mark success/failure
134+
# when "tearDown" task with trigger rule is part of the DAG
135+
list(dag.tasks) >> watcher()
101136

102137

103138
from tests.system.utils import get_test_run # noqa: E402

‎tests/system/providers/google/cloud/life_sciences/resources/__init__.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.

‎tests/system/providers/google/cloud/life_sciences/resources/file

Whitespace-only changes.

0 commit comments

Comments
 (0)








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: https://github.com/apache/airflow/commit/3a539ff6631109dc58514339ac60672f031c7054

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy