From cbabbe432517b457851e709e6fc512b041df79bb Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 23 Dec 2025 11:45:33 +0100 Subject: [PATCH 1/5] set API workers to a default fo 1 --- docs/modules/airflow/pages/troubleshooting/index.adoc | 6 +++--- rust/operator-binary/src/env_vars.rs | 11 +++++++++++ .../kuttl/logging/41-install-airflow-cluster.yaml.j2 | 5 ----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/modules/airflow/pages/troubleshooting/index.adoc b/docs/modules/airflow/pages/troubleshooting/index.adoc index c796bb4c..f89d5438 100644 --- a/docs/modules/airflow/pages/troubleshooting/index.adoc +++ b/docs/modules/airflow/pages/troubleshooting/index.adoc @@ -29,8 +29,8 @@ See e.g. https://github.com/minio/minio/issues/20845[this MinIO issue] for detai == Setting API Workers In Airflow the webserver (called the API Server in Airflow 3.x+) can use multiple workers. -This is determined by the environment variable `+AIRFLOW__API__WORKERS+` and is set by default to `4`. -For most cases the default should work without problem, but if you run into performance issues and would like to add more workers, you can either modulate multiple worker processes at the level of webserver, keeping the default value for each one: +This is determined by the environment variable `+AIRFLOW__API__WORKERS+` and is set by default to `1`. +For most cases this should work without problem, but if you run into performance issues and would like to add more workers, you can either modulate multiple worker processes at the level of webserver (keeping the default value for each instance): [source,yaml] ---- @@ -46,7 +46,7 @@ or change the environment variable using `envOverrides`: ---- webservers: envOverrides: - AIRFLOW__API__WORKERS: "6" # something other than the default + AIRFLOW__API__WORKERS: "4" # something other than the default ---- TIP: Our strong recommendation is to increase the webserver replicas, with each webserver running a single worker, as this removes the risk of running into timeouts or memory issues. diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index b04144bc..28a96a71 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -505,6 +505,17 @@ fn add_version_specific_env_vars( JWT_SECRET_SECRET_KEY, ), ); + // The airflow default for this is 4, but we document a recommended + // value of 1 with the option of either increasing it via overrides + // or of adding webserver instances. + env.insert( + "AIRFLOW__API__WORKERS".into(), + EnvVar { + name: "AIRFLOW__API__WORKERS".into(), + value: Some("1".into()), + ..Default::default() + }, + ); if airflow_role == &AirflowRole::Webserver { // Sometimes a race condition can arise when both scheduler and // api-server are updating the DB, which adds overhead (conflicts diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index 40e027aa..498f6db0 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -84,11 +84,6 @@ spec: max: 2000m memory: limit: 3Gi - envOverrides: - # logging tests use two webservers and if two tests should run in - # parallel then the CPU usage can be high if the default number of - # workers (4) is used - AIRFLOW__API__WORKERS: "1" roleGroups: automatic-log-config: replicas: 1 From 47592d55c0643b3923a8456ad952c00df0a7942c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 23 Dec 2025 12:32:18 +0100 Subject: [PATCH 2/5] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb30599f..170839ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +### Changed + +- Re-set `API_WORKERS` default to 1 for Airflow 3.x and update/correct docs to reflect this ([#727]). + +[#727]: https://github.com/stackabletech/airflow-operator/pull/727 + ## [25.11.0] - 2025-11-07 ## [25.11.0-rc1] - 2025-11-06 From 8d8cfb432bda2e675c3282064bf94fbd82135488 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:08:06 +0100 Subject: [PATCH 3/5] Update rust/operator-binary/src/env_vars.rs Co-authored-by: Sebastian Bernauer --- rust/operator-binary/src/env_vars.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index 28a96a71..92960842 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -505,9 +505,10 @@ fn add_version_specific_env_vars( JWT_SECRET_SECRET_KEY, ), ); - // The airflow default for this is 4, but we document a recommended - // value of 1 with the option of either increasing it via overrides - // or of adding webserver instances. + // The Airflow default for this is 4. + // However, with the default resources this could cause problems, + // as the Pod went to 100% CPU usage and didn't get healthy + // quick enough, resulting in a crashloop. env.insert( "AIRFLOW__API__WORKERS".into(), EnvVar { From fd00e6b13c49f95066b2a88bf765517ce3101c1d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:08:19 +0100 Subject: [PATCH 4/5] Update CHANGELOG.md Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170839ec..8f59a114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -### Changed +### Fixed - Re-set `API_WORKERS` default to 1 for Airflow 3.x and update/correct docs to reflect this ([#727]). From d496e636f94c7a915421c7a267e972c0fff2a828 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:08:34 +0100 Subject: [PATCH 5/5] Update CHANGELOG.md Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f59a114..bd2495d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Fixed -- Re-set `API_WORKERS` default to 1 for Airflow 3.x and update/correct docs to reflect this ([#727]). +- Default `API_WORKERS` to 1 (instead of letting Airflow default to 4) to prevent crashloop and update/correct docs to reflect this ([#727]). [#727]: https://github.com/stackabletech/airflow-operator/pull/727