diff --git a/CHANGELOG.md b/CHANGELOG.md index aeb468ef..03b51aea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- BREAKING: Add required CLI argument and env var to set the image repository used to construct final product image names: `IMAGE_REPOSITORY` (`--image-repository`), eg. `oci.example.org/my/namespace` ([#784]). - Support for passing CAs to GitSync ([#750]). - Support setting `clientAuthenticationMethod` for OIDC authentication. The value is passed through to the Flask-AppBuilder config as `token_endpoint_auth_method` ([#765]). @@ -31,6 +32,7 @@ [#775]: https://github.com/stackabletech/airflow-operator/pull/775 [#777]: https://github.com/stackabletech/airflow-operator/pull/777 [#754]: https://github.com/stackabletech/airflow-operator/pull/754 +[#784]: https://github.com/stackabletech/airflow-operator/pull/784 ## [26.3.0] - 2026-03-16 diff --git a/Cargo.lock b/Cargo.lock index 3c4d4890..d6e8310c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1525,7 +1525,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "darling", "regex", @@ -2510,9 +2510,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -2920,7 +2920,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "const-oid", "ecdsa", @@ -2943,8 +2943,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.110.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +version = "0.111.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "base64", "clap", @@ -2985,7 +2985,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "darling", "proc-macro2", @@ -2996,7 +2996,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "jiff", "k8s-openapi", @@ -3013,7 +3013,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "axum", "clap", @@ -3036,9 +3036,10 @@ dependencies = [ [[package]] name = "stackable-versioned" -version = "0.9.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +version = "0.10.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ + "kube", "schemars", "serde", "serde_json", @@ -3049,8 +3050,8 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" -version = "0.9.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +version = "0.10.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "convert_case", "convert_case_extras", @@ -3068,7 +3069,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#96f42571ea185a3cd76fedde351fcabbeefcae16" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 939286fb..32f0dd35 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4867,7 +4867,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "k8s_version"; @@ -8372,9 +8372,9 @@ rec { }; "rustls-webpki" = rec { crateName = "rustls-webpki"; - version = "0.103.12"; + version = "0.103.13"; edition = "2021"; - sha256 = "01nxzkfd1l96jzp04svc7iznlkarzx3wb9p63a0i17rc4y2vnyc2"; + sha256 = "0vkm7z9pnxz5qz66p2kmyy2pwx0g4jnsbqk5xzfhs4czcjl2ki31"; libName = "webpki"; dependencies = [ { @@ -9642,7 +9642,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_certs"; @@ -9740,12 +9740,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.110.1"; + version = "0.111.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_operator"; @@ -9925,7 +9925,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; procMacro = true; @@ -9960,7 +9960,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_shared"; @@ -10041,7 +10041,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_telemetry"; @@ -10146,12 +10146,12 @@ rec { }; "stackable-versioned" = rec { crateName = "stackable-versioned"; - version = "0.9.0"; + version = "0.10.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_versioned"; @@ -10159,6 +10159,12 @@ rec { "Stackable GmbH " ]; dependencies = [ + { + name = "kube"; + packageId = "kube"; + usesDefaultFeatures = false; + features = [ "client" "jsonpatch" "runtime" "derive" "admission" "rustls-tls" "ring" ]; + } { name = "schemars"; packageId = "schemars"; @@ -10190,12 +10196,12 @@ rec { }; "stackable-versioned-macros" = rec { crateName = "stackable-versioned-macros"; - version = "0.9.0"; + version = "0.10.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; procMacro = true; @@ -10263,7 +10269,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "96f42571ea185a3cd76fedde351fcabbeefcae16"; + rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; sha256 = "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by"; }; libName = "stackable_webhook"; diff --git a/Cargo.toml b/Cargo.toml index a7feba3e..d04c1f1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ product-config = { git = "https://github.com/stackabletech/product-config.git", stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = [ "crds", "webhook", -], tag = "stackable-operator-0.110.1" } +], tag = "stackable-operator-0.111.0" } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index e19b553d..4ab79fa6 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#k8s-version@0.1.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-certs@0.4.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator-derive@0.3.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-operator@0.110.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-shared@0.1.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-telemetry@0.6.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned-macros@0.9.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-versioned@0.9.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.110.1#stackable-webhook@0.9.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "0d58yvxvy8hbai12bjhcyvh4zw182j5dsfyqja4k2xc1vzjy29by", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index ac6f6dba..f195a1bf 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2576,8 +2576,9 @@ spec: properties: custom: description: |- - Overwrite the docker image. - Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` + Provide a custom container image. + + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` type: string productVersion: description: Version of the product, e.g. `1.4.1`. @@ -2604,14 +2605,20 @@ spec: nullable: true type: array repo: - description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` + description: |- + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. nullable: true type: string stackableVersion: description: |- Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. - If not specified, the operator will use its own version, e.g. `23.4.1`. - When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. nullable: true type: string type: object @@ -8735,8 +8742,9 @@ spec: properties: custom: description: |- - Overwrite the docker image. - Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` + Provide a custom container image. + + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` type: string productVersion: description: Version of the product, e.g. `1.4.1`. @@ -8763,14 +8771,20 @@ spec: nullable: true type: array repo: - description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` + description: |- + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. nullable: true type: string stackableVersion: description: |- Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. - If not specified, the operator will use its own version, e.g. `23.4.1`. - When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. nullable: true type: string type: object diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index b6d95f48..25a5bc71 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -29,6 +29,7 @@ use stackable_operator::{ }, }, }, + cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ product_image_selection::{self, ResolvedProductImage}, @@ -119,13 +120,14 @@ use crate::{ }; pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; -pub const DOCKER_IMAGE_BASE_NAME: &str = "airflow"; +pub const CONTAINER_IMAGE_BASE_NAME: &str = "airflow"; pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = concatcp!(AIRFLOW_CONTROLLER_NAME, '.', OPERATOR_NAME); pub struct Ctx { pub client: stackable_operator::client::Client, pub product_config: ProductConfigManager, + pub operator_environment: OperatorEnvironmentOptions, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -387,7 +389,11 @@ pub async fn reconcile_airflow( let resolved_product_image = airflow .spec .image - .resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::PKG_VERSION) + .resolve( + CONTAINER_IMAGE_BASE_NAME, + &ctx.operator_environment.image_repository, + crate::built_info::PKG_VERSION, + ) .context(ResolveProductImageSnafu)?; let cluster_operation_cond_builder = diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e3eedf6a..f0fc0f27 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1110,9 +1110,13 @@ pub fn build_recommended_labels<'a, T>( #[cfg(test)] mod tests { - use stackable_operator::commons::product_image_selection::ResolvedProductImage; + use indoc::formatdoc; + use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + versioned::test_utils::RoundtripTestData, + }; - use crate::v1alpha2::AirflowCluster; + use crate::{v1alpha1, v1alpha2}; #[test] fn test_cluster_config() { @@ -1146,13 +1150,13 @@ mod tests { "; let deserializer = serde_yaml::Deserializer::from_str(cluster); - let cluster: AirflowCluster = + let cluster: v1alpha2::AirflowCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let resolved_airflow_image: ResolvedProductImage = cluster .spec .image - .resolve("airflow", "0.0.0-dev") + .resolve("airflow", "oci.example.org", "0.0.0-dev") .expect("test: resolved product image is always valid"); assert_eq!("3.1.6", &resolved_airflow_image.product_version); @@ -1166,4 +1170,175 @@ mod tests { // defaults to true assert!(cluster.spec.cluster_config.database_initialization.enabled); } + + impl RoundtripTestData for v1alpha1::AirflowClusterSpec { + fn roundtrip_test_data() -> Vec { + let git_sync_section = r#" + dagsGitSync: + - repo: ssh://git@github.com/stackable-airflow/dags.git + - repo: https://github.com/stackable-airflow/dags + branch: main + credentialsSecret: my-basic-auth + wait: 5s + gitSyncConf: + foo: bar + gitFolder: "mount-dags-gitsync/dags_airflow3" + tls: + verification: + server: + caCert: + secretClass: git-ca-cert + - repo: ssh://git@github.com/stackable-airflow/dags.git + "#; + let yaml = test_airflow_cluster_yaml(git_sync_section); + stackable_operator::utils::yaml_from_str_singleton_map(&yaml) + .expect("Failed to parse AirflowClusterSpec YAML") + } + } + + // FIXME: The roundtrip looses data when private keys are used. + // See https://github.com/stackabletech/issues/issues/849 for details. + impl RoundtripTestData for v1alpha2::AirflowClusterSpec { + fn roundtrip_test_data() -> Vec { + let git_sync_section = r#" + dagsGitSync: + - repo: ssh://git@github.com/stackable-airflow/dags.git + - repo: https://github.com/stackable-airflow/dags + branch: main + credentials: + basicAuthSecretName: my-basic-auth + wait: 5s + gitSyncConf: + foo: bar + gitFolder: "mount-dags-gitsync/dags_airflow3" + tls: + verification: + server: + caCert: + secretClass: git-ca-cert + - repo: ssh://git@github.com/stackable-airflow/dags.git + # FIXME: The roundtrip looses data when private keys are used. + # See https://github.com/stackabletech/issues/issues/849 for details. + # credentials: + # sshPrivateKeySecretName: my-private-key + "#; + let yaml = test_airflow_cluster_yaml(git_sync_section); + stackable_operator::utils::yaml_from_str_singleton_map(&yaml) + .expect("Failed to parse AirflowClusterSpec YAML") + } + } + + fn test_airflow_cluster_yaml(git_sync_section: &str) -> String { + formatdoc! { + r#" + - image: + productVersion: 1.2.3 + pullPolicy: IfNotPresent + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery + loadExamples: true + credentialsSecretName: airflow-admin-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + authentication: + - authenticationClass: my-ldap + userRegistrationRole: Admin + authorization: + opa: + configMapName: test-opa + package: airflow + cache: + entryTimeToLive: 5s + maxEntries: 10 + volumes: + - name: test-cm-dag + configMap: + name: test-cm-dag + volumeMounts: + - name: test-cm-dag + mountPath: /dags/example_trigger_target_dag.py + subPath: example_trigger_target_dag.py + {git_sync_section} + webservers: + roleConfig: + listenerClass: external-unstable + config: + resources: + cpu: + min: 100m + max: "1" + memory: + limit: 3001Mi + logging: + enableVectorAgent: true + configOverrides: + webserver_config.py: + FILE_HEADER: | + COMMON_HEADER_VAR = "role-value" + ROLE_HEADER_VAR = "role-value" + FILE_FOOTER: | + ROLE_FOOTER_VAR = "role-value" + podOverrides: + spec: + containers: + - name: airflow + resources: + requests: + cpu: 300m + limits: + cpu: 900m + roleGroups: + default: + replicas: 1 + configOverrides: + webserver_config.py: + FILE_HEADER: | + COMMON_HEADER_VAR = "group-value" + celeryExecutors: + resultBackend: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + broker: + redis: + host: airflow-redis-master + credentialsSecretName: airflow-redis-credentials + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 2 + kubernetesExecutors: + config: + logging: + enableVectorAgent: true + schedulers: + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 1 + dagProcessors: + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 1 + triggerers: + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 1 + "# + } + } } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 4abf38c3..6cc89e99 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -121,6 +121,17 @@ async fn main() -> anyhow::Result<()> { }, )); + let webhook_server = create_webhook_server( + &operator_environment, + maintenance.disable_crd_maintenance, + client.as_kube_client(), + ) + .await?; + + let webhook_server = webhook_server + .run(sigterm_watcher.handle()) + .map_err(|err| anyhow!(err).context("failed to run webhook server")); + let airflow_controller = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default(), @@ -170,6 +181,7 @@ async fn main() -> anyhow::Result<()> { airflow_controller::error_policy, Arc::new(airflow_controller::Ctx { client: client.clone(), + operator_environment, product_config, }), ) @@ -192,17 +204,6 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); - let webhook_server = create_webhook_server( - &operator_environment, - maintenance.disable_crd_maintenance, - client.as_kube_client(), - ) - .await?; - - let webhook_server = webhook_server - .run(sigterm_watcher.handle()) - .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let delayed_airflow_controller = async { signal::crd_established(&client, v1alpha1::AirflowCluster::crd_name(), None) .await?;