From 3f7ca038801e4261b2245f32a5807cdaef1b5f4c Mon Sep 17 00:00:00 2001 From: kubevirt-bot Date: Thu, 27 Feb 2025 13:00:48 +0100 Subject: [PATCH] [release-0.97] Align placement with HCO components (#2011) * placement: Remove default Infra tolerations To follow the rest of HCO components this change remove the default Infra tolerations configuration, openshift configure already default ones that are used by kubevirt and CDI. Signed-off-by: Enrique Llorente * placement: Add unschedulable key to Workload After cordon a node we get: taints: - effect: NoSchedule key: node.kubernetes.io/unschedulable timeAdded: "2025-02-26T08:18:29Z" This change add the missing "key" value at the toleration to be more specific. Signed-off-by: Enrique Llorente * placement: Use infra config for control plane pods CNAO is using Workload placement config for deployments that are part of the control plane, this end up them running at worker nodes and also propertly preventing drain the node. This change move those to Infra placement so they will run at control plane nodes. Signed-off-by: Enrique Llorente --------- Signed-off-by: Enrique Llorente Co-authored-by: Enrique Llorente --- .../003-passtbindingcni.yaml | 6 ++-- .../bump-kubevirt-ipam-controller.sh | 6 ++-- pkg/network/kube_secondary_dns_controller.go | 2 +- pkg/network/kubevirt_ipam_controller.go | 3 +- pkg/network/kubevirt_ipam_controller_test.go | 5 +++- pkg/network/placement_configuration.go | 28 ++----------------- 6 files changed, 15 insertions(+), 35 deletions(-) diff --git a/data/kubevirt-ipam-controller/003-passtbindingcni.yaml b/data/kubevirt-ipam-controller/003-passtbindingcni.yaml index a4384d0fc..60d188d30 100644 --- a/data/kubevirt-ipam-controller/003-passtbindingcni.yaml +++ b/data/kubevirt-ipam-controller/003-passtbindingcni.yaml @@ -52,6 +52,6 @@ spec: - name: cnibin hostPath: path: {{ .CNIBinDir }} - nodeSelector: {{ toYaml .Placement.NodeSelector | nindent 8 }} - affinity: {{ toYaml .Placement.Affinity | nindent 8 }} - tolerations: {{ toYaml .Placement.Tolerations | nindent 8 }} + nodeSelector: {{ toYaml .PlacementPasst.NodeSelector | nindent 8 }} + affinity: {{ toYaml .PlacementPasst.Affinity | nindent 8 }} + tolerations: {{ toYaml .PlacementPasst.Tolerations | nindent 8 }} diff --git a/hack/components/bump-kubevirt-ipam-controller.sh b/hack/components/bump-kubevirt-ipam-controller.sh index 09a249205..ca7e22bd9 100755 --- a/hack/components/bump-kubevirt-ipam-controller.sh +++ b/hack/components/bump-kubevirt-ipam-controller.sh @@ -75,9 +75,9 @@ function __parametize_passt_binding_cni() { f=003-passtbindingcni.yaml yaml-utils::update_param ${f} metadata.namespace '{{ .Namespace }}' - yaml-utils::set_param ${f} spec.template.spec.nodeSelector '{{ toYaml .Placement.NodeSelector | nindent 8 }}' - yaml-utils::set_param ${f} spec.template.spec.affinity '{{ toYaml .Placement.Affinity | nindent 8 }}' - yaml-utils::set_param ${f} spec.template.spec.tolerations '{{ toYaml .Placement.Tolerations | nindent 8 }}' + yaml-utils::set_param ${f} spec.template.spec.nodeSelector '{{ toYaml .PlacementPasst.NodeSelector | nindent 8 }}' + yaml-utils::set_param ${f} spec.template.spec.affinity '{{ toYaml .PlacementPasst.Affinity | nindent 8 }}' + yaml-utils::set_param ${f} spec.template.spec.tolerations '{{ toYaml .PlacementPasst.Tolerations | nindent 8 }}' yaml-utils::update_param ${f} spec.template.spec.containers[0].image '{{ .PasstBindingCNIImage }}' yaml-utils::set_param ${f} spec.template.spec.containers[0].imagePullPolicy '{{ .ImagePullPolicy }}' yaml-utils::update_param ${f} spec.template.spec.volumes[0].hostPath.path '{{ .CNIBinDir }}' diff --git a/pkg/network/kube_secondary_dns_controller.go b/pkg/network/kube_secondary_dns_controller.go index 3a86bbf98..5b2f1821d 100644 --- a/pkg/network/kube_secondary_dns_controller.go +++ b/pkg/network/kube_secondary_dns_controller.go @@ -21,7 +21,7 @@ func renderKubeSecondaryDNS(conf *cnao.NetworkAddonsConfigSpec, manifestDir stri data := render.MakeRenderData() data.Data["Namespace"] = os.Getenv("OPERAND_NAMESPACE") data.Data["ImagePullPolicy"] = conf.ImagePullPolicy - data.Data["Placement"] = conf.PlacementConfiguration.Workloads + data.Data["Placement"] = conf.PlacementConfiguration.Infra data.Data["Domain"] = conf.KubeSecondaryDNS.Domain data.Data["NameServerIp"] = conf.KubeSecondaryDNS.NameServerIP data.Data["KubeSecondaryDNSImage"] = os.Getenv("KUBE_SECONDARY_DNS_IMAGE") diff --git a/pkg/network/kubevirt_ipam_controller.go b/pkg/network/kubevirt_ipam_controller.go index ee6cf90a0..aba5bceac 100644 --- a/pkg/network/kubevirt_ipam_controller.go +++ b/pkg/network/kubevirt_ipam_controller.go @@ -29,7 +29,8 @@ func renderKubevirtIPAMController(conf *cnao.NetworkAddonsConfigSpec, manifestDi data := render.MakeRenderData() data.Data["Namespace"] = os.Getenv("OPERAND_NAMESPACE") data.Data["ImagePullPolicy"] = conf.ImagePullPolicy - data.Data["Placement"] = conf.PlacementConfiguration.Workloads + data.Data["Placement"] = conf.PlacementConfiguration.Infra + data.Data["PlacementPasst"] = conf.PlacementConfiguration.Workloads data.Data["KubevirtIpamControllerImage"] = os.Getenv("KUBEVIRT_IPAM_CONTROLLER_IMAGE") data.Data["PasstBindingCNIImage"] = os.Getenv("PASST_BINDING_CNI_IMAGE") diff --git a/pkg/network/kubevirt_ipam_controller_test.go b/pkg/network/kubevirt_ipam_controller_test.go index eeccac1a8..8701793ab 100644 --- a/pkg/network/kubevirt_ipam_controller_test.go +++ b/pkg/network/kubevirt_ipam_controller_test.go @@ -14,7 +14,10 @@ import ( var _ = Describe("Testing kubevirt ipam controller", func() { Context("Render KubevirtIpamController", func() { - conf := &cnao.NetworkAddonsConfigSpec{ImagePullPolicy: v1.PullAlways, Multus: &cnao.Multus{}, KubevirtIpamController: &cnao.KubevirtIpamController{}, PlacementConfiguration: &cnao.PlacementConfiguration{Workloads: &cnao.Placement{}}} + conf := &cnao.NetworkAddonsConfigSpec{ImagePullPolicy: v1.PullAlways, Multus: &cnao.Multus{}, KubevirtIpamController: &cnao.KubevirtIpamController{}, PlacementConfiguration: &cnao.PlacementConfiguration{ + Infra: &cnao.Placement{}, + Workloads: &cnao.Placement{}, + }} manifestDir := "../../data" openshiftNetworkConf := &osv1.Network{} clusterInfo := &ClusterInfo{} diff --git a/pkg/network/placement_configuration.go b/pkg/network/placement_configuration.go index 8d71bc86c..6f37fc0ee 100644 --- a/pkg/network/placement_configuration.go +++ b/pkg/network/placement_configuration.go @@ -7,33 +7,8 @@ import ( ) func GetDefaultPlacementConfiguration() cnao.PlacementConfiguration { - var nodeNotNReadyTolerationInSeconds int64 = 60 return cnao.PlacementConfiguration{ Infra: &cnao.Placement{ - Tolerations: []corev1.Toleration{ - corev1.Toleration{ - Key: "node-role.kubernetes.io/control-plane", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }, - corev1.Toleration{ - Key: "node-role.kubernetes.io/master", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }, - corev1.Toleration{ - Key: "node.kubernetes.io/unreachable", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoExecute, - TolerationSeconds: &nodeNotNReadyTolerationInSeconds, - }, - corev1.Toleration{ - Key: "node.kubernetes.io/not-ready", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoExecute, - TolerationSeconds: &nodeNotNReadyTolerationInSeconds, - }, - }, Affinity: corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ @@ -65,10 +40,11 @@ func GetDefaultPlacementConfiguration() cnao.PlacementConfiguration { }, Workloads: &cnao.Placement{ NodeSelector: map[string]string{ - "kubernetes.io/os": "linux", + corev1.LabelOSStable: "linux", }, Tolerations: []corev1.Toleration{ corev1.Toleration{ + Key: corev1.TaintNodeUnschedulable, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, },