Content-Length: 997599 | pFad | https://github.com/apache/airflow/commit/1059de6294cf89a085c02036d117000bd98435dc

24 Dataform operators, links, update system tests and docs (#27144) · apache/airflow@1059de6 · GitHub
Skip to content

Commit 1059de6

Browse files
MrGeorgeOwlMrGeorgeOwl
authored
Dataform operators, links, update system tests and docs (#27144)
1 parent 0a059ee commit 1059de6

File tree

10 files changed

+2183
-32
lines changed

10 files changed

+2183
-32
lines changed

‎airflow/providers/google/cloud/hooks/dataform.py

Lines changed: 377 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,14 @@
2222
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
2323
from google.api_core.retry import Retry
2424
from google.cloud.dataform_v1beta1 import DataformClient
25-
from google.cloud.dataform_v1beta1.types import CompilationResult, WorkflowInvocation
25+
from google.cloud.dataform_v1beta1.types import (
26+
CompilationResult,
27+
InstallNpmPackagesResponse,
28+
Repository,
29+
WorkflowInvocation,
30+
Workspace,
31+
WriteFileResponse,
32+
)
2633

2734
from airflow import AirflowException
2835
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
@@ -249,3 +256,372 @@ def cancel_workflow_invocation(
249256
client.cancel_workflow_invocation(
250257
request={"name": name}, retry=retry, timeout=timeout, metadata=metadata
251258
)
259+
260+
@GoogleBaseHook.fallback_to_default_project_id
261+
def create_repository(
262+
self,
263+
*,
264+
project_id: str,
265+
region: str,
266+
repository_id: str,
267+
retry: Retry | _MethodDefault = DEFAULT,
268+
timeout: float | None = None,
269+
metadata: Sequence[tuple[str, str]] = (),
270+
) -> Repository:
271+
"""
272+
Creates repository
273+
274+
:param project_id: Required. The ID of the Google Cloud project where repository should be.
275+
:param region: Required. The ID of the Google Cloud region where repository should be.
276+
:param repository_id: Required. The ID of the new Dataform repository.
277+
:param retry: Designation of what errors, if any, should be retried.
278+
:param timeout: The timeout for this request.
279+
:param metadata: Strings which should be sent along with the request as metadata.
280+
"""
281+
client = self.get_dataform_client()
282+
parent = f"projects/{project_id}/locations/{region}"
283+
request = {
284+
"parent": parent,
285+
"repository_id": repository_id,
286+
}
287+
288+
repository = client.create_repository(
289+
request=request,
290+
retry=retry,
291+
timeout=timeout,
292+
metadata=metadata,
293+
)
294+
295+
return repository
296+
297+
@GoogleBaseHook.fallback_to_default_project_id
298+
def delete_repository(
299+
self,
300+
*,
301+
project_id: str,
302+
region: str,
303+
repository_id: str,
304+
force: bool = True,
305+
retry: Retry | _MethodDefault = DEFAULT,
306+
timeout: float | None = None,
307+
metadata: Sequence[tuple[str, str]] = (),
308+
) -> None:
309+
"""
310+
Deletes repository.
311+
312+
:param project_id: Required. The ID of the Google Cloud project where repository located.
313+
:param region: Required. The ID of the Google Cloud region where repository located.
314+
:param repository_id: Required. The ID of the Dataform repository that should be deleted.
315+
:param force: If set to true, any child resources of this repository will also be deleted.
316+
:param retry: Designation of what errors, if any, should be retried.
317+
:param timeout: The timeout for this request.
318+
:param metadata: Strings which should be sent along with the request as metadata.
319+
"""
320+
client = self.get_dataform_client()
321+
name = f"projects/{project_id}/locations/{region}/repositories/{repository_id}"
322+
request = {
323+
"name": name,
324+
"force": force,
325+
}
326+
327+
client.delete_repository(
328+
request=request,
329+
retry=retry,
330+
timeout=timeout,
331+
metadata=metadata,
332+
)
333+
334+
@GoogleBaseHook.fallback_to_default_project_id
335+
def create_workspace(
336+
self,
337+
*,
338+
project_id: str,
339+
region: str,
340+
repository_id: str,
341+
workspace_id: str,
342+
retry: Retry | _MethodDefault = DEFAULT,
343+
timeout: float | None = None,
344+
metadata: Sequence[tuple[str, str]] = (),
345+
) -> Workspace:
346+
"""
347+
Creates workspace.
348+
349+
:param project_id: Required. The ID of the Google Cloud project where workspace should be.
350+
:param region: Required. The ID of the Google Cloud region where workspace should be.
351+
:param repository_id: Required. The ID of the Dataform repository where workspace should be.
352+
:param workspace_id: Required. The ID of the new Dataform workspace.
353+
:param retry: Designation of what errors, if any, should be retried.
354+
:param timeout: The timeout for this request.
355+
:param metadata: Strings which should be sent along with the request as metadata.
356+
"""
357+
client = self.get_dataform_client()
358+
parent = f"projects/{project_id}/locations/{region}/repositories/{repository_id}"
359+
360+
request = {"parent": parent, "workspace_id": workspace_id}
361+
362+
workspace = client.create_workspace(
363+
request=request,
364+
retry=retry,
365+
timeout=timeout,
366+
metadata=metadata,
367+
)
368+
369+
return workspace
370+
371+
@GoogleBaseHook.fallback_to_default_project_id
372+
def delete_workspace(
373+
self,
374+
*,
375+
project_id: str,
376+
region: str,
377+
repository_id: str,
378+
workspace_id: str,
379+
retry: Retry | _MethodDefault = DEFAULT,
380+
timeout: float | None = None,
381+
metadata: Sequence[tuple[str, str]] = (),
382+
):
383+
"""
384+
Deletes workspace.
385+
386+
:param project_id: Required. The ID of the Google Cloud project where workspace located.
387+
:param region: Required. The ID of the Google Cloud region where workspace located.
388+
:param repository_id: Required. The ID of the Dataform repository where workspace located.
389+
:param workspace_id: Required. The ID of the Dataform workspace that should be deleted.
390+
:param retry: Designation of what errors, if any, should be retried.
391+
:param timeout: The timeout for this request.
392+
:param metadata: Strings which should be sent along with the request as metadata.
393+
"""
394+
client = self.get_dataform_client()
395+
workspace_path = (
396+
f"projects/{project_id}/locations/{region}/"
397+
f"repositories/{repository_id}/workspaces/{workspace_id}"
398+
)
399+
request = {
400+
"name": workspace_path,
401+
}
402+
403+
client.delete_workspace(
404+
request=request,
405+
retry=retry,
406+
timeout=timeout,
407+
metadata=metadata,
408+
)
409+
410+
@GoogleBaseHook.fallback_to_default_project_id
411+
def write_file(
412+
self,
413+
*,
414+
project_id: str,
415+
region: str,
416+
repository_id: str,
417+
workspace_id: str,
418+
filepath: str,
419+
contents: bytes,
420+
retry: Retry | _MethodDefault = DEFAULT,
421+
timeout: float | None = None,
422+
metadata: Sequence[tuple[str, str]] = (),
423+
) -> WriteFileResponse:
424+
"""
425+
Writes a new file to the specified workspace.
426+
427+
:param project_id: Required. The ID of the Google Cloud project where workspace located.
428+
:param region: Required. The ID of the Google Cloud region where workspace located.
429+
:param repository_id: Required. The ID of the Dataform repository where workspace located.
430+
:param workspace_id: Required. The ID of the Dataform workspace where files should be created.
431+
:param filepath: Required. Path to file including name of the file relative to workspace root.
432+
:param contents: Required. Content of the file to be written.
433+
:param retry: Designation of what errors, if any, should be retried.
434+
:param timeout: The timeout for this request.
435+
:param metadata: Strings which should be sent along with the request as metadata.
436+
"""
437+
client = self.get_dataform_client()
438+
workspace_path = (
439+
f"projects/{project_id}/locations/{region}/"
440+
f"repositories/{repository_id}/workspaces/{workspace_id}"
441+
)
442+
request = {
443+
"workspace": workspace_path,
444+
"path": filepath,
445+
"contents": contents,
446+
}
447+
448+
response = client.write_file(
449+
request=request,
450+
retry=retry,
451+
timeout=timeout,
452+
metadata=metadata,
453+
)
454+
455+
return response
456+
457+
@GoogleBaseHook.fallback_to_default_project_id
458+
def make_directory(
459+
self,
460+
*,
461+
project_id: str,
462+
region: str,
463+
repository_id: str,
464+
workspace_id: str,
465+
path: str,
466+
retry: Retry | _MethodDefault = DEFAULT,
467+
timeout: float | None = None,
468+
metadata: Sequence[tuple[str, str]] = (),
469+
) -> dict:
470+
"""
471+
Makes new directory in specified workspace.
472+
473+
:param project_id: Required. The ID of the Google Cloud project where workspace located.
474+
:param region: Required. The ID of the Google Cloud region where workspace located.
475+
:param repository_id: Required. The ID of the Dataform repository where workspace located.
476+
:param workspace_id: Required. The ID of the Dataform workspace where directory should be created.
477+
:param path: Required. The directory's full path including new directory name,
478+
relative to the workspace root.
479+
:param retry: Designation of what errors, if any, should be retried.
480+
:param timeout: The timeout for this request.
481+
:param metadata: Strings which should be sent along with the request as metadata.
482+
"""
483+
client = self.get_dataform_client()
484+
workspace_path = (
485+
f"projects/{project_id}/locations/{region}/"
486+
f"repositories/{repository_id}/workspaces/{workspace_id}"
487+
)
488+
request = {
489+
"workspace": workspace_path,
490+
"path": path,
491+
}
492+
493+
response = client.make_directory(
494+
request=request,
495+
retry=retry,
496+
timeout=timeout,
497+
metadata=metadata,
498+
)
499+
500+
return response
501+
502+
@GoogleBaseHook.fallback_to_default_project_id
503+
def remove_directory(
504+
self,
505+
*,
506+
project_id: str,
507+
region: str,
508+
repository_id: str,
509+
workspace_id: str,
510+
path: str,
511+
retry: Retry | _MethodDefault = DEFAULT,
512+
timeout: float | None = None,
513+
metadata: Sequence[tuple[str, str]] = (),
514+
):
515+
"""
516+
Removes directory in specified workspace.
517+
518+
:param project_id: Required. The ID of the Google Cloud project where workspace located.
519+
:param region: Required. The ID of the Google Cloud region where workspace located.
520+
:param repository_id: Required. The ID of the Dataform repository where workspace located.
521+
:param workspace_id: Required. The ID of the Dataform workspace where directory located.
522+
:param path: Required. The directory's full path including directory name,
523+
relative to the workspace root.
524+
:param retry: Designation of what errors, if any, should be retried.
525+
:param timeout: The timeout for this request.
526+
:param metadata: Strings which should be sent along with the request as metadata.
527+
"""
528+
client = self.get_dataform_client()
529+
workspace_path = (
530+
f"projects/{project_id}/locations/{region}/"
531+
f"repositories/{repository_id}/workspaces/{workspace_id}"
532+
)
533+
request = {
534+
"workspace": workspace_path,
535+
"path": path,
536+
}
537+
538+
client.remove_directory(
539+
request=request,
540+
retry=retry,
541+
timeout=timeout,
542+
metadata=metadata,
543+
)
544+
545+
@GoogleBaseHook.fallback_to_default_project_id
546+
def remove_file(
547+
self,
548+
*,
549+
project_id: str,
550+
region: str,
551+
repository_id: str,
552+
workspace_id: str,
553+
filepath: str,
554+
retry: Retry | _MethodDefault = DEFAULT,
555+
timeout: float | None = None,
556+
metadata: Sequence[tuple[str, str]] = (),
557+
):
558+
"""
559+
Removes file in specified workspace.
560+
561+
:param project_id: Required. The ID of the Google Cloud project where workspace located.
562+
:param region: Required. The ID of the Google Cloud region where workspace located.
563+
:param repository_id: Required. The ID of the Dataform repository where workspace located.
564+
:param workspace_id: Required. The ID of the Dataform workspace where directory located.
565+
:param filepath: Required. The full path including name of the file, relative to the workspace root.
566+
:param retry: Designation of what errors, if any, should be retried.
567+
:param timeout: The timeout for this request.
568+
:param metadata: Strings which should be sent along with the request as metadata.
569+
"""
570+
client = self.get_dataform_client()
571+
workspace_path = (
572+
f"projects/{project_id}/locations/{region}/"
573+
f"repositories/{repository_id}/workspaces/{workspace_id}"
574+
)
575+
request = {
576+
"workspace": workspace_path,
577+
"path": filepath,
578+
}
579+
580+
client.remove_file(
581+
request=request,
582+
retry=retry,
583+
timeout=timeout,
584+
metadata=metadata,
585+
)
586+
587+
@GoogleBaseHook.fallback_to_default_project_id
588+
def install_npm_packages(
589+
self,
590+
*,
591+
project_id: str,
592+
region: str,
593+
repository_id: str,
594+
workspace_id: str,
595+
retry: Retry | _MethodDefault = DEFAULT,
596+
timeout: float | None = None,
597+
metadata: Sequence[tuple[str, str]] = (),
598+
) -> InstallNpmPackagesResponse:
599+
"""
600+
Installs npm dependencies in the provided workspace. Requires "package.json"
601+
to be created in workspace
602+
603+
:param project_id: Required. The ID of the Google Cloud project where workspace located.
604+
:param region: Required. The ID of the Google Cloud region where workspace located.
605+
:param repository_id: Required. The ID of the Dataform repository where workspace located.
606+
:param workspace_id: Required. The ID of the Dataform workspace.
607+
:param retry: Designation of what errors, if any, should be retried.
608+
:param timeout: The timeout for this request.
609+
:param metadata: Strings which should be sent along with the request as metadata.
610+
"""
611+
client = self.get_dataform_client()
612+
workspace_path = (
613+
f"projects/{project_id}/locations/{region}/"
614+
f"repositories/{repository_id}/workspaces/{workspace_id}"
615+
)
616+
request = {
617+
"workspace": workspace_path,
618+
}
619+
620+
response = client.install_npm_packages(
621+
request=request,
622+
retry=retry,
623+
timeout=timeout,
624+
metadata=metadata,
625+
)
626+
627+
return response

0 commit comments

Comments
 (0)








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: https://github.com/apache/airflow/commit/1059de6294cf89a085c02036d117000bd98435dc

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy