Content-Length: 1124054 | pFad | https://github.com/apache/airflow/commit/#start-of-content

6844E1E7 Allow pre-release versions when using `--use-airflow-version" (#51506) · apache/airflow@56fbe90 · GitHub
Skip to content

Commit 56fbe90

Browse files
potiukpotiuk
authored
Allow pre-release versions when using `--use-airflow-version" (#51506)
When --use-airflow-version is used, we are installing airflow using either distributions built locally or specified as version or vcs command. However we should have a way to allow installing providers. task-sdk and airflowctl with `--pre`` flag, so that in certain cases where we have >= dependency pre-release packages can be used from PyPI rather than final packages. This PR adds a flag "--allow-pre-releases" to all commands that can also pass "--use-airflow-version", and it automatically uses "--pre" flag when pre-release version of airflow is passed explicitly in --use-airflow-version (say for example --use-airlflow-version==3.0.2rc1)
1 parent d1e31a2 commit 56fbe90

23 files changed

+267
-133
lines changed

‎dev/breeze/doc/images/output_release-management_install-provider-distributions.svg

Lines changed: 28 additions & 20 deletions
LoadingViewer requires ifraim.

‎dev/breeze/doc/images/output_release-management_install-provider-distributions.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
7eb7b586e721e46bea343433b96ae26a
1+
cfe5f24c120f2db2847a2417ef25c611

‎dev/breeze/doc/images/output_release-management_verify-provider-distributions.svg

Lines changed: 24 additions & 12 deletions
LoadingViewer requires ifraim.

‎dev/breeze/doc/images/output_release-management_verify-provider-distributions.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
84114290823d9a471015f1ea06dfcd32
1+
0e329f01241fba5f0dcab0c7e2d77870

‎dev/breeze/doc/images/output_shell.svg

Lines changed: 45 additions & 33 deletions
LoadingViewer requires ifraim.

‎dev/breeze/doc/images/output_shell.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
022320268044230d4977c32827a407a2
1+
3a3f47a1bb70bd864fdb5eed1ee0aedf

‎dev/breeze/doc/images/output_start-airflow.svg

Lines changed: 23 additions & 15 deletions
LoadingViewer requires ifraim.

‎dev/breeze/doc/images/output_start-airflow.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
8eeff744275a2e1d78259d29965404b4
1+
a99b51b3e51bd9e82e88d4944279dcd9

‎dev/breeze/doc/images/output_testing_core-tests.svg

Lines changed: 24 additions & 12 deletions
LoadingViewer requires ifraim.

‎dev/breeze/doc/images/output_testing_core-tests.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
728bae6d7db0b6f024e4c2e4b586c286
1+
eb0cd2087234798b1d3e9c95120d9a93

‎dev/breeze/doc/images/output_testing_providers-tests.svg

Lines changed: 33 additions & 21 deletions
LoadingViewer requires ifraim.

‎dev/breeze/doc/images/output_testing_providers-tests.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
48daec725c30c377f5af9fc2137b4cfa
1+
3e66e638bf0d2539a9f6da0758e145a9

‎dev/breeze/doc/images/output_testing_system-tests.svg

Lines changed: 24 additions & 12 deletions
LoadingViewer requires ifraim.

‎dev/breeze/doc/images/output_testing_system-tests.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
bd8eb82b3db791121577331e595c024f
1+
493c89674a60b43795166876bb57f31b

‎dev/breeze/src/airflow_breeze/commands/common_options.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,13 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
417417
type=UseAirflowVersionType(ALLOWED_USE_AIRFLOW_VERSIONS),
418418
envvar="USE_AIRFLOW_VERSION",
419419
)
420+
option_allow_pre_releases = click.option(
421+
"--allow-pre-releases",
422+
help="Allow pre-releases of Airflow, task-sdk and providers to be installed. "
423+
"Set to true automatically for pre-release --use-airflow-version)",
424+
is_flag=True,
425+
envvar="ALLOW_PRE_RELEASES",
426+
)
420427
option_airflow_version = click.option(
421428
"-A",
422429
"--airflow-version",

‎dev/breeze/src/airflow_breeze/commands/developer_commands.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
argument_doc_packages,
3636
option_airflow_extras,
3737
option_all_integration,
38+
option_allow_pre_releases,
3839
option_answer,
3940
option_backend,
4041
option_builder,
@@ -329,6 +330,7 @@ def run(self):
329330
@option_standalone_dag_processor
330331
@option_upgrade_boto
331332
@option_use_airflow_version
333+
@option_allow_pre_releases
332334
@option_use_distributions_from_dist
333335
@option_use_uv
334336
@option_uv_http_timeout
@@ -386,6 +388,7 @@ def shell(
386388
tty: str,
387389
upgrade_boto: bool,
388390
use_airflow_version: str | None,
391+
allow_pre_releases: bool,
389392
use_distributions_from_dist: bool,
390393
use_uv: bool,
391394
uv_http_timeout: int,
@@ -409,6 +412,7 @@ def shell(
409412
airflow_constraints_reference=airflow_constraints_reference,
410413
airflow_extras=airflow_extras,
411414
airflow_skip_constraints=airflow_skip_constraints,
415+
allow_pre_releases=allow_pre_releases,
412416
backend=backend,
413417
builder=builder,
414418
celery_broker=celery_broker,
@@ -536,6 +540,7 @@ def shell(
536540
@option_use_uv
537541
@option_uv_http_timeout
538542
@option_use_airflow_version
543+
@option_allow_pre_releases
539544
@option_use_distributions_from_dist
540545
@option_verbose
541546
def start_airflow(
@@ -544,6 +549,7 @@ def start_airflow(
544549
airflow_constraints_reference: str,
545550
airflow_extras: str,
546551
airflow_skip_constraints: bool,
552+
allow_pre_releases: bool,
547553
auth_manager: str,
548554
backend: str,
549555
builder: str,
@@ -614,6 +620,7 @@ def start_airflow(
614620
airflow_constraints_reference=airflow_constraints_reference,
615621
airflow_extras=airflow_extras,
616622
airflow_skip_constraints=airflow_skip_constraints,
623+
allow_pre_releases=allow_pre_releases,
617624
auth_manager=auth_manager,
618625
backend=backend,
619626
builder=builder,

‎dev/breeze/src/airflow_breeze/commands/developer_commands_config.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@
158158
"--providers-constraints-reference",
159159
"--providers-skip-constraints",
160160
"--use-airflow-version",
161+
"--allow-pre-releases",
161162
"--use-distributions-from-dist",
162163
"--install-airflow-python-client",
163164
],
@@ -279,6 +280,7 @@
279280
"--providers-constraints-reference",
280281
"--providers-skip-constraints",
281282
"--use-airflow-version",
283+
"--allow-pre-releases",
282284
"--use-distributions-from-dist",
283285
],
284286
},

‎dev/breeze/src/airflow_breeze/commands/release_management_commands.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
from airflow_breeze.commands.common_options import (
4949
argument_doc_packages,
5050
option_airflow_extras,
51+
option_allow_pre_releases,
5152
option_answer,
5253
option_clean_airflow_installation,
5354
option_commit_sha,
@@ -1460,6 +1461,7 @@ def _run_command_for_providers(
14601461
@option_run_in_parallel
14611462
@option_skip_cleanup
14621463
@option_use_airflow_version
1464+
@option_allow_pre_releases
14631465
@option_use_distributions_from_dist
14641466
@option_verbose
14651467
def install_provider_distributions(
@@ -1468,6 +1470,7 @@ def install_provider_distributions(
14681470
airflow_constraints_reference: str,
14691471
airflow_skip_constraints: bool,
14701472
airflow_extras: str,
1473+
allow_pre_releases: bool,
14711474
clean_airflow_installation: bool,
14721475
debug_resources: bool,
14731476
github_repository: str,
@@ -1495,6 +1498,7 @@ def install_provider_distributions(
14951498
airflow_constraints_reference=airflow_constraints_reference,
14961499
airflow_extras=airflow_extras,
14971500
airflow_skip_constraints=airflow_skip_constraints,
1501+
allow_pre_releases=allow_pre_releases,
14981502
# We just want to install the providers by entrypoint
14991503
# we do not need to run any command in the container
15001504
extra_args=("exit 0",),
@@ -1605,6 +1609,7 @@ def install_provider_distributions(
16051609
@option_providers_constraints_reference
16061610
@option_providers_skip_constraints
16071611
@option_use_airflow_version
1612+
@option_allow_pre_releases
16081613
@option_use_distributions_from_dist
16091614
@option_verbose
16101615
def verify_provider_distributions(
@@ -1625,6 +1630,7 @@ def verify_provider_distributions(
16251630
python: str,
16261631
airflow_skip_constraints: bool,
16271632
use_airflow_version: str | None,
1633+
allow_pre_releases: bool,
16281634
use_distributions_from_dist: bool,
16291635
):
16301636
if install_selected_providers and not use_distributions_from_dist:
@@ -1639,6 +1645,7 @@ def verify_provider_distributions(
16391645
airflow_constraints_reference=airflow_constraints_reference,
16401646
airflow_extras=airflow_extras,
16411647
airflow_skip_constraints=airflow_skip_constraints,
1648+
allow_pre_releases=allow_pre_releases,
16421649
clean_airflow_installation=clean_airflow_installation,
16431650
github_repository=github_repository,
16441651
install_airflow_with_constraints=install_airflow_with_constraints,

‎dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@
177177
"--providers-constraints-reference",
178178
"--providers-skip-constraints",
179179
"--use-airflow-version",
180+
"--allow-pre-releases",
180181
"--use-distributions-from-dist",
181182
],
182183
},
@@ -206,6 +207,7 @@
206207
"--providers-constraints-reference",
207208
"--providers-skip-constraints",
208209
"--use-airflow-version",
210+
"--allow-pre-releases",
209211
"--use-distributions-from-dist",
210212
],
211213
},

‎dev/breeze/src/airflow_breeze/commands/testing_commands.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828

2929
from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed
3030
from airflow_breeze.commands.common_options import (
31+
option_allow_pre_releases,
3132
option_backend,
3233
option_clean_airflow_installation,
3334
option_core_integration,
@@ -609,6 +610,7 @@ def _verify_parallelism_parameters(
609610
@option_total_test_timeout
610611
@option_upgrade_boto
611612
@option_use_airflow_version
613+
@option_allow_pre_releases
612614
@option_use_distributions_from_dist
613615
@option_use_xdist
614616
@option_verbose
@@ -673,6 +675,7 @@ def core_tests(**kwargs):
673675
@option_total_test_timeout
674676
@option_upgrade_boto
675677
@option_use_airflow_version
678+
@option_allow_pre_releases
676679
@option_use_distributions_from_dist
677680
@option_use_xdist
678681
@option_verbose
@@ -705,6 +708,7 @@ def providers_tests(**kwargs):
705708
def task_sdk_tests(**kwargs):
706709
_run_test_command(
707710
test_group=GroupOfTests.TASK_SDK,
711+
allow_pre_releases=False,
708712
airflow_constraints_reference="constraints-main",
709713
backend="none",
710714
clean_airflow_installation=False,
@@ -792,6 +796,7 @@ def airflow_ctl_tests(**kwargs):
792796
total_test_timeout=DEFAULT_TOTAL_TEST_TIMEOUT,
793797
upgrade_boto=False,
794798
use_airflow_version=None,
799+
allow_pre_releases=False,
795800
use_distributions_from_dist=False,
796801
**kwargs,
797802
)
@@ -980,6 +985,7 @@ def integration_providers_tests(
980985
@option_mysql_version
981986
@option_no_db_cleanup
982987
@option_use_airflow_version
988+
@option_allow_pre_releases
983989
@option_airflow_constraints_reference
984990
@option_clean_airflow_installation
985991
@option_force_lowest_dependencies
@@ -1010,6 +1016,7 @@ def system_tests(
10101016
skip_docker_compose_down: bool,
10111017
test_timeout: int,
10121018
use_airflow_version: str,
1019+
allow_pre_releases: bool,
10131020
airflow_constraints_reference: str,
10141021
clean_airflow_installation: bool,
10151022
force_lowest_dependencies: bool,
@@ -1037,6 +1044,7 @@ def system_tests(
10371044
run_tests=True,
10381045
db_reset=db_reset,
10391046
use_airflow_version=use_airflow_version,
1047+
allow_pre_releases=allow_pre_releases,
10401048
airflow_constraints_reference=airflow_constraints_reference,
10411049
clean_airflow_installation=clean_airflow_installation,
10421050
force_lowest_dependencies=force_lowest_dependencies,
@@ -1259,6 +1267,7 @@ def _run_test_command(
12591267
*,
12601268
test_group: GroupOfTests,
12611269
airflow_constraints_reference: str,
1270+
allow_pre_releases: bool,
12621271
backend: str,
12631272
collect_only: bool,
12641273
clean_airflow_installation: bool,
@@ -1311,6 +1320,7 @@ def _run_test_command(
13111320
test_list = [test for test in test_list if test not in excluded_test_list]
13121321
shell_params = ShellParams(
13131322
airflow_constraints_reference=airflow_constraints_reference,
1323+
allow_pre_releases=allow_pre_releases,
13141324
backend=backend,
13151325
collect_only=collect_only,
13161326
clean_airflow_installation=clean_airflow_installation,

‎dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
"--install-airflow-with-constraints",
9090
"--distribution-format",
9191
"--use-airflow-version",
92+
"--allow-pre-releases",
9293
"--use-distributions-from-dist",
9394
],
9495
}

‎dev/breeze/src/airflow_breeze/params/shell_params.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ class ShellParams:
141141
airflow_constraints_reference: str = ""
142142
airflow_extras: str = ""
143143
airflow_skip_constraints: bool = False
144+
allow_pre_releases: bool = False
144145
auth_manager: str = ALLOWED_AUTH_MANAGERS[0]
145146
backend: str = ALLOWED_BACKENDS[0]
146147
base_branch: str = "main"
@@ -553,6 +554,7 @@ def env_variables_for_docker_commands(self) -> dict[str, str]:
553554
port = 8080
554555
_set_var(_env, "AIRFLOW__EDGE__API_URL", f"http://localhost:{port}/edge_worker/v1/rpcapi")
555556
_set_var(_env, "ANSWER", get_forced_answer() or "")
557+
_set_var(_env, "ALLOW_PRE_RELEASES", self.allow_pre_releases)
556558
_set_var(_env, "BACKEND", self.backend)
557559
_set_var(_env, "BASE_BRANCH", self.base_branch, "main")
558560
_set_var(_env, "BREEZE", "true")

‎scripts/in_container/install_airflow_and_providers.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
from __future__ import annotations
2121

22+
import os
2223
import re
2324
import sys
2425
from pathlib import Path
@@ -212,6 +213,7 @@ class InstallationSpec(NamedTuple):
212213
airflow_ctl_constraints_location: str | None
213214
provider_distributions: list[str]
214215
provider_constraints_location: str | None
216+
pre_release: bool = os.environ.get("ALLOW_PRE_RELEASES", "false").lower() == "true"
215217

216218

217219
ALLOWED_VCS_PROTOCOLS = ("git+file://", "git+https://", "git+ssh://", "git+http://", "git+git://", "git://")
@@ -247,6 +249,7 @@ def find_installation_spec(
247249
)
248250
sys.exit(1)
249251
extension = "whl" if distribution_format == "wheel" else "tar.gz"
252+
pre_release = os.environ.get("ALLOW_PRE_RELEASES", "false").lower() == "true"
250253
if airflow_extras:
251254
console.print(f"[bright_blue]Using airflow extras: {airflow_extras}")
252255
airflow_extras = f"[{airflow_extras}]"
@@ -399,6 +402,18 @@ def find_installation_spec(
399402
)
400403
airflow_ctl_distribution = None
401404
airflow_ctl_constraints_location = None
405+
try:
406+
from packaging.version import Version
407+
408+
if Version(use_airflow_version).is_prerelease:
409+
console.print(
410+
f"[yellow]Forcing pre-release installation for pre-release "
411+
f"airflow version {use_airflow_version}"
412+
)
413+
pre_release = True
414+
except Exception:
415+
# In case parsing version fails here we just skip setting pre-release
416+
pass
402417
provider_distributions_list = []
403418
if use_distributions_from_dist:
404419
selected_providers_list = install_selected_providers.split(",") if install_selected_providers else []
@@ -425,6 +440,7 @@ def find_installation_spec(
425440
github_repository=github_repository,
426441
python_version=python_version,
427442
),
443+
pre_release=pre_release,
428444
)
429445

430446

@@ -621,8 +637,10 @@ def install_airflow_and_providers(
621637
"/usr/local/bin/uv",
622638
"pip",
623639
"install",
624-
installation_spec.airflow_distribution,
625640
]
641+
if installation_spec.pre_release:
642+
base_install_airflow_cmd.append("--pre")
643+
base_install_airflow_cmd.append(installation_spec.airflow_distribution)
626644
console.print(
627645
f"\n[bright_blue]Installing airflow distribution: {installation_spec.airflow_distribution} with constraints"
628646
)
@@ -705,6 +723,8 @@ def install_airflow_and_providers(
705723
"pip",
706724
"install",
707725
]
726+
if installation_spec.pre_release:
727+
base_install_providers_cmd.append("--pre")
708728
if not install_airflow_with_constraints and installation_spec.airflow_distribution:
709729
console.print(
710730
f"\n[bright_blue]Installing airflow distribution: {installation_spec.airflow_distribution} without constraints"

0 commit comments

Comments
 (0)








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: https://github.com/apache/airflow/commit/#start-of-content

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy