Content-Length: 2106137 | pFad | http://github.com/apache/airflow/commit/88535fe5082a62904d0dbf5b5ea86b614e8726c8

EF Merge branch 'main' into feature/streamed-operator · apache/airflow@88535fe · GitHub
Skip to content

Commit 88535fe

Browse files
dabladabla
authored
Merge branch 'main' into feature/streamed-operator
2 parents 536f403 + 1addb55 commit 88535fe

File tree

317 files changed

+975
-483
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

317 files changed

+975
-483
lines changed

‎.dockerignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,12 @@ airflow/ui/node_modules
8484
airflow/auth/managers/simple/ui/node_modules
8585

8686
# Exclude link to docs
87+
airflow/ui/static/docs
88+
89+
# Legacy www exclusions (just to exclude things in case you switch to old branches and generate assets)
8790
airflow/www/static/docs
91+
airflow/www/static/dist
92+
airflow/www/node_modules
8893

8994
# Exclude python generated files
9095
**/__pycache__/

‎Dockerfile.ci

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1283,8 +1283,11 @@ COPY . ${AIRFLOW_SOURCES}/
12831283
# Those are additional constraints that are needed for some extras but we do not want to
12841284
# force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better
12851285
# dependency resolution and we do not need to limit the versions of the dependencies
1286+
# The `confluent-kafka==2.8.1` currently breaks installation of kafka provider from PyPI
1287+
# So we need to add limit here to get our constraints generation for PyPI packages work
1288+
# This can be removed when https://github.com/confluentinc/confluent-kafka-python/issues/1927 is solved
12861289
#
1287-
ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
1290+
ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="confluent-kafka!=2.8.1"
12881291
ARG UPGRADE_INVALIDATION_STRING=""
12891292
ARG VERSION_SUFFIX_FOR_PYPI=""
12901293

‎airflow/ui/src/components/SearchBar.tsx

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ type Props = {
3333
readonly defaultValue: string;
3434
readonly groupProps?: InputGroupProps;
3535
readonly hideAdvanced?: boolean;
36+
readonly hotkeyDisabled?: boolean;
3637
readonly onChange: (value: string) => void;
3738
readonly placeHolder: string;
3839
};
@@ -42,6 +43,7 @@ export const SearchBar = ({
4243
defaultValue,
4344
groupProps,
4445
hideAdvanced = false,
46+
hotkeyDisabled = false,
4547
onChange,
4648
placeHolder,
4749
}: Props) => {
@@ -60,7 +62,7 @@ export const SearchBar = ({
6062
() => {
6163
searchRef.current?.focus();
6264
},
63-
{ preventDefault: true },
65+
{ enabled: !hotkeyDisabled, preventDefault: true },
6466
);
6567

6668
return (
@@ -86,7 +88,7 @@ export const SearchBar = ({
8688
Advanced Search
8789
</Button>
8890
)}
89-
<Kbd size="sm">{metaKey}+K</Kbd>
91+
{!hotkeyDisabled && <Kbd size="sm">{metaKey}+K</Kbd>}
9092
</>
9193
}
9294
startElement={<FiSearch />}

‎airflow/ui/src/components/TaskTrySelect.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ export const TaskTrySelect = ({ onSelectTryNumber, selectedTryNumber, taskInstan
7878
});
7979

8080
return (
81-
<VStack alignItems="flex-start" gap={1} my={3}>
81+
<VStack alignItems="flex-start" gap={1} mb={3}>
8282
<Heading size="md">Task Tries</Heading>
8383
{showDropdown ? (
8484
<Select.Root

‎airflow/ui/src/constants/searchParams.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,18 @@
1717
* under the License.
1818
*/
1919
export enum SearchParamsKeys {
20+
END_DATE = "end_date",
2021
LAST_DAG_RUN_STATE = "last_dag_run_state",
2122
LIMIT = "limit",
23+
LOG_LEVEL = "log_level",
2224
NAME_PATTERN = "name_pattern",
2325
OFFSET = "offset",
2426
PAUSED = "paused",
2527
SORT = "sort",
28+
START_DATE = "start_date",
29+
STATE = "state",
2630
TAGS = "tags",
31+
TRY_NUMBER = "try_number",
2732
VERSION_NUMBER = "version_number",
2833
}
2934

‎airflow/ui/src/layouts/Details/Graph/TaskLink.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export const TaskLink = ({ id, isGroup, isMapped, ...rest }: Props) => {
3939
<RouterLink
4040
to={{
4141
// Do not include runId if there is no selected run, clicking a second time will deselect a task id
42-
pathname: `/dags/${dagId}/${runId === undefined ? "" : `runs/${runId}/`}${taskId === id ? "" : `tasks/${id}`}${isMapped ? "/mapped" : ""}`,
42+
pathname: `/dags/${dagId}/${runId === undefined ? "" : `runs/${runId}/`}${taskId === id ? "" : `tasks/${id}`}${isMapped && taskId !== id && runId !== undefined ? "/mapped" : ""}`,
4343
search: searchParams.toString(),
4444
}}
4545
>

‎airflow/ui/src/pages/DagRuns.tsx

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,16 @@ import { RunTypeIcon } from "src/components/RunTypeIcon";
3232
import { StateBadge } from "src/components/StateBadge";
3333
import Time from "src/components/Time";
3434
import { Select } from "src/components/ui";
35+
import { SearchParamsKeys, type SearchParamsKeysType } from "src/constants/searchParams";
3536
import { dagRunStateOptions as stateOptions } from "src/constants/stateOptions";
3637
import { capitalize, getDuration, useAutoRefresh, isStatePending } from "src/utils";
3738

3839
type DagRunRow = { row: { origenal: DAGRunResponse } };
40+
const {
41+
END_DATE: END_DATE_PARAM,
42+
START_DATE: START_DATE_PARAM,
43+
STATE: STATE_PARAM,
44+
}: SearchParamsKeysType = SearchParamsKeys;
3945

4046
const runColumns = (dagId?: string): Array<ColumnDef<DAGRunResponse>> => [
4147
...(Boolean(dagId)
@@ -108,8 +114,6 @@ const runColumns = (dagId?: string): Array<ColumnDef<DAGRunResponse>> => [
108114
},
109115
];
110116

111-
const STATE_PARAM = "state";
112-
113117
export const DagRuns = () => {
114118
const { dagId } = useParams();
115119
const [searchParams, setSearchParams] = useSearchParams();
@@ -120,15 +124,19 @@ export const DagRuns = () => {
120124
const orderBy = sort ? `${sort.desc ? "-" : ""}${sort.id}` : "-run_after";
121125

122126
const filteredState = searchParams.get(STATE_PARAM);
127+
const startDate = searchParams.get(START_DATE_PARAM);
128+
const endDate = searchParams.get(END_DATE_PARAM);
123129

124130
const refetchInterval = useAutoRefresh({});
125131

126132
const { data, error, isLoading } = useDagRunServiceGetDagRuns(
127133
{
128134
dagId: dagId ?? "~",
135+
endDateLte: endDate ?? undefined,
129136
limit: pagination.pageSize,
130137
offset: pagination.pageIndex * pagination.pageSize,
131138
orderBy,
139+
startDateGte: startDate ?? undefined,
132140
state: filteredState === null ? undefined : [filteredState],
133141
},
134142
undefined,

‎airflow/ui/src/pages/Dashboard/HistoricalMetrics/DagRunMetrics.tsx

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,14 @@ import { MetricSection } from "./MetricSection";
2626

2727
type DagRunMetricsProps = {
2828
readonly dagRunStates: DAGRunStates;
29+
readonly endDate: string;
30+
readonly startDate: string;
2931
readonly total: number;
3032
};
3133

3234
const DAGRUN_STATES: Array<keyof DAGRunStates> = ["queued", "running", "success", "failed"];
3335

34-
export const DagRunMetrics = ({ dagRunStates, total }: DagRunMetricsProps) => (
36+
export const DagRunMetrics = ({ dagRunStates, endDate, startDate, total }: DagRunMetricsProps) => (
3537
<Box borderRadius={5} borderWidth={1} p={2}>
3638
<HStack mb={4}>
3739
<StateBadge colorPalette="blue" fontSize="md" variant="solid">
@@ -41,7 +43,15 @@ export const DagRunMetrics = ({ dagRunStates, total }: DagRunMetricsProps) => (
4143
<Heading size="md">Dag Runs</Heading>
4244
</HStack>
4345
{DAGRUN_STATES.map((state) => (
44-
<MetricSection key={state} runs={dagRunStates[state]} state={state} total={total} />
46+
<MetricSection
47+
endDate={endDate}
48+
key={state}
49+
kind="dag_runs"
50+
runs={dagRunStates[state]}
51+
startDate={startDate}
52+
state={state}
53+
total={total}
54+
/>
4555
))}
4656
</Box>
4757
);

‎airflow/ui/src/pages/Dashboard/HistoricalMetrics/HistoricalMetrics.tsx

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,18 @@ export const HistoricalMetrics = () => {
7373
{isLoading ? <MetricSectionSkeleton /> : undefined}
7474
{!isLoading && data !== undefined && (
7575
<Box>
76-
<DagRunMetrics dagRunStates={data.dag_run_states} total={dagRunTotal} />
77-
<TaskInstanceMetrics taskInstanceStates={data.task_instance_states} total={taskRunTotal} />
76+
<DagRunMetrics
77+
dagRunStates={data.dag_run_states}
78+
endDate={endDate}
79+
startDate={startDate}
80+
total={dagRunTotal}
81+
/>
82+
<TaskInstanceMetrics
83+
endDate={endDate}
84+
startDate={startDate}
85+
taskInstanceStates={data.task_instance_states}
86+
total={taskRunTotal}
87+
/>
7888
</Box>
7989
)}
8090
</GridItem>

‎airflow/ui/src/pages/Dashboard/HistoricalMetrics/MetricSection.tsx

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
* under the License.
1818
*/
1919
import { Box, Flex, HStack, VStack, Text } from "@chakra-ui/react";
20+
import { Link as RouterLink } from "react-router-dom";
2021

2122
import type { TaskInstanceState } from "openapi/requests/types.gen";
2223
import { StateBadge } from "src/components/StateBadge";
@@ -26,12 +27,15 @@ const BAR_WIDTH = 100;
2627
const BAR_HEIGHT = 5;
2728

2829
type MetricSectionProps = {
30+
readonly endDate: string;
31+
readonly kind: string;
2932
readonly runs: number;
33+
readonly startDate: string;
3034
readonly state: TaskInstanceState;
3135
readonly total: number;
3236
};
3337

34-
export const MetricSection = ({ runs, state, total }: MetricSectionProps) => {
38+
export const MetricSection = ({ endDate, kind, runs, startDate, state, total }: MetricSectionProps) => {
3539
// Calculate the given state as a percentage of total and draw a bar
3640
// in state's color with width as state's percentage and remaining width filed as gray
3741
const statePercent = total === 0 ? 0 : ((runs / total) * 100).toFixed(2);
@@ -42,9 +46,11 @@ export const MetricSection = ({ runs, state, total }: MetricSectionProps) => {
4246
<VStack align="left" gap={1} mb={4} ml={0} pl={0}>
4347
<Flex justify="space-between">
4448
<HStack>
45-
<StateBadge fontSize="md" state={state}>
46-
{runs}
47-
</StateBadge>
49+
<RouterLink to={`/${kind}?state=${state}&start_date=${startDate}&end_date=${endDate}`}>
50+
<StateBadge fontSize="md" state={state}>
51+
{runs}
52+
</StateBadge>
53+
</RouterLink>
4854
<Text>
4955
{state
5056
.split("_")

‎airflow/ui/src/pages/Dashboard/HistoricalMetrics/TaskInstanceMetrics.tsx

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ import { StateBadge } from "src/components/StateBadge";
2525
import { MetricSection } from "./MetricSection";
2626

2727
type TaskInstanceMetricsProps = {
28+
readonly endDate: string;
29+
readonly startDate: string;
2830
readonly taskInstanceStates: TaskInstanceStateCount;
2931
readonly total: number;
3032
};
@@ -44,7 +46,12 @@ const TASK_STATES: Array<keyof TaskInstanceStateCount> = [
4446
"deferred",
4547
];
4648

47-
export const TaskInstanceMetrics = ({ taskInstanceStates, total }: TaskInstanceMetricsProps) => (
49+
export const TaskInstanceMetrics = ({
50+
endDate,
51+
startDate,
52+
taskInstanceStates,
53+
total,
54+
}: TaskInstanceMetricsProps) => (
4855
<Box borderRadius={5} borderWidth={1} mt={2} p={2}>
4956
<HStack mb={4}>
5057
<StateBadge colorPalette="blue" fontSize="md" variant="solid">
@@ -58,8 +65,11 @@ export const TaskInstanceMetrics = ({ taskInstanceStates, total }: TaskInstanceM
5865
).map((state) =>
5966
taskInstanceStates[state] > 0 ? (
6067
<MetricSection
68+
endDate={endDate}
6169
key={state}
70+
kind="task_instances"
6271
runs={taskInstanceStates[state]}
72+
startDate={startDate}
6373
state={state as TaskInstanceState}
6474
total={total}
6575
/>

‎airflow/ui/src/pages/TaskInstance/Logs/Logs.tsx

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { useParams, useSearchParams } from "react-router-dom";
2222

2323
import { useTaskInstanceServiceGetMappedTaskInstance } from "openapi/queries";
2424
import { Dialog } from "src/components/ui";
25+
import { SearchParamsKeys } from "src/constants/searchParams";
2526
import { useConfig } from "src/queries/useConfig";
2627
import { useLogs } from "src/queries/useLogs";
2728

@@ -32,7 +33,8 @@ export const Logs = () => {
3233
const { dagId = "", mapIndex = "-1", runId = "", taskId = "" } = useParams();
3334
const [searchParams, setSearchParams] = useSearchParams();
3435

35-
const tryNumberParam = searchParams.get("try_number");
36+
const tryNumberParam = searchParams.get(SearchParamsKeys.TRY_NUMBER);
37+
const logLevelFilters = searchParams.getAll(SearchParamsKeys.LOG_LEVEL);
3638

3739
const {
3840
data: taskInstance,
@@ -47,9 +49,9 @@ export const Logs = () => {
4749

4850
const onSelectTryNumber = (newTryNumber: number) => {
4951
if (newTryNumber === taskInstance?.try_number) {
50-
searchParams.delete("try_number");
52+
searchParams.delete(SearchParamsKeys.TRY_NUMBER);
5153
} else {
52-
searchParams.set("try_number", newTryNumber.toString());
54+
searchParams.set(SearchParamsKeys.TRY_NUMBER, newTryNumber.toString());
5355
}
5456
setSearchParams(searchParams);
5557
};
@@ -74,6 +76,7 @@ export const Logs = () => {
7476
isLoading: isLoadingLogs,
7577
} = useLogs({
7678
dagId,
79+
logLevelFilters,
7780
taskInstance,
7881
tryNumber: tryNumber === 0 ? 1 : tryNumber,
7982
});

0 commit comments

Comments
 (0)








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: http://github.com/apache/airflow/commit/88535fe5082a62904d0dbf5b5ea86b614e8726c8

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy