AGENT-1072: Apply operator manifests from ConfigMap for agent-installer

In the agent-based installer, when cluster installation is complete the
assisted-installer cannot access assisted-service since it has
terminated. Therefore any operator custom-manifests are not applied.
Instead the assisted-service will create a ConfigMap with the
manifests and operator info (openshift/assisted-service#7818)
and in this PR the assisted-installer will retrieve the ConfigMap and
apply the manifests.

The intent is that this will only be done for the agent-based installer
and not affect the normal path of applying manifests using the
assisted-service API.

Author: bfournie

src/assisted_installer_controller/assisted_installer_controller.go

@@ -38,6 +38,7 @@ import (
 	"github.com/openshift/assisted-installer/src/ops"
 	"github.com/openshift/assisted-installer/src/utils"
 	"github.com/openshift/assisted-service/models"
+	"gopkg.in/yaml.v2"
 )
 
 const (
@@ -112,6 +113,7 @@ type Controller interface {
 	WaitAndUpdateNodesStatus(ctx context.Context, wg *sync.WaitGroup, removeUninitializedTaint bool)
 	HackDNSAddressConflict(wg *sync.WaitGroup)
 	PostInstallConfigs(ctx context.Context, wg *sync.WaitGroup)
+	PostInstallConfigsK8sClient(ctx context.Context, wg *sync.WaitGroup, kubeconfigPath string)
 	UpdateNodeLabels(ctx context.Context, wg *sync.WaitGroup)
 	UpdateBMHs(ctx context.Context, wg *sync.WaitGroup)
 	UploadLogs(ctx context.Context, wg *sync.WaitGroup, invoker string)
@@ -129,14 +131,21 @@ type controller struct {
 	rebootsNotifier RebootsNotifier
 }
 
-// manifest store the operator manifest used by assisted-installer to create CRs of the OLM:
-type manifest struct {
+// Manifest stores the operator manifest used by assisted-installer to create CRs of the OLM:
+type Manifest struct {
 	// name of the operator the CR manifest we want create
 	Name string
-	// content of the manifest of the opreator
+	// content of the manifest of the operator
 	Content string
 }
 
+// Contents of ConfigMap containing operator manifests
+type operatorMetadata struct {
+	Namespace        string   `yaml:"namespace"`
+	SubscriptionName string   `yaml:"subscriptionName"`
+	Manifests        []string `yaml:"manifests"`
+}
+
 func newController(log *logrus.Logger, cfg ControllerConfig, ops ops.Ops, ic inventory_client.InventoryClient, kc k8s_client.K8SClient, rebootsNotifier RebootsNotifier) *controller {
 	return &controller{
 		log:              log,
@@ -487,6 +496,73 @@ func (c *controller) PostInstallConfigs(ctx context.Context, wg *sync.WaitGroup)
 	c.sendCompleteInstallation(ctx, success, errMessage, data)
 }
 
+func (c *controller) PostInstallConfigsK8sClient(ctx context.Context, wg *sync.WaitGroup, kubeconfigPath string) {
+	defer func() {
+		c.log.Infof("Finished PostInstallConfigsK8sClient")
+		wg.Done()
+	}()
+	err := utils.WaitForeverForPredicate(ctx, GeneralWaitInterval, func() bool {
+		// Get operator info from ConfigMap created by assisted-service
+		cm, err := c.kc.GetConfigMap("assisted-installer", "olm-operator-manifests")
+		if err != nil {
+			c.log.Infof("Failed to get ConfigMap %s", err.Error())
+			return true // configmap may not have been created
+		}
+
+		var manifests []Manifest
+		var olmOperators []models.MonitoredOperator
+		// Iterate through the ConfigMap's Data to find metadata and manifests
+		for key, value := range cm.Data {
+			if strings.HasSuffix(key, ".metadata.yaml") {
+				// This is an operator metadata entry
+				var metadata operatorMetadata
+				if err := yaml.Unmarshal([]byte(value), &metadata); err != nil {
+					c.log.Infof("Failed to unmarshal metadata for %s: %s", key, err.Error())
+					return false
+				}
+
+				// Extract the operator name from the metadata key
+				operatorName := strings.TrimSuffix(key, ".metadata.yaml")
+
+				// Get manifests associated with this operator
+				for _, manifestFileName := range metadata.Manifests {
+					if content, found := cm.Data[manifestFileName]; found {
+						manifests = append(manifests, Manifest{operatorName, content})
+					} else {
+						c.log.Infof("Manifest file %s listed in metadata but not found in ConfigMap data", manifestFileName)
+					}
+				}
+
+				olmOperators = append(olmOperators, models.MonitoredOperator{
+					Name:             operatorName,
+					Namespace:        metadata.Namespace,
+					SubscriptionName: metadata.SubscriptionName,
+					OperatorType:     models.OperatorTypeOlm,
+				})
+			}
+		}
+
+		if kubeconfigPath == "" {
+			kubeconfigPath, err = c.rebootsNotifier.GetKubeconfigPath(ctx)
+			if err != nil {
+				c.log.Infof("Failed to get kubeconfig path %s", err.Error())
+				return false
+			}
+		}
+		// Apply manifests from configMap if ready
+		if !c.applyOperatorManifests(olmOperators, manifests, kubeconfigPath) {
+			c.log.Infof("Failed to apply manifests")
+		} else {
+			c.log.Infof("Successfully applied manifests")
+			return true
+		}
+		return false
+	})
+	if err != nil {
+		c.log.Infof("Error in wait %s", err.Error())
+	}
+}
+
 func (c *controller) postInstallConfigs(ctx context.Context) (error, map[string]interface{}) {
 	var err error
 	var data map[string]interface{}
@@ -650,7 +726,7 @@ func (c *controller) applyPostInstallManifests(operators []models.MonitoredOpera
 	}
 
 	// Unmarshall the content of the operators manifests:
-	var manifests []manifest
+	var manifests []Manifest
 	data, err := os.ReadFile(customManifestPath)
 	if err != nil {
 		c.log.WithError(err).Errorf("Failed to read the custom manifests file.")
@@ -661,10 +737,24 @@ func (c *controller) applyPostInstallManifests(operators []models.MonitoredOpera
 		return false
 	}
 
-	// Create the manifests of the operators, which are properly initialized:
+	if !c.applyOperatorManifests(operators, manifests, kubeconfigName) {
+		c.log.WithError(err).Errorf("Failed to apply operators to cluster")
+		return false
+	}
+
+	return true
+}
+
+// Apply the operator manifests to the cluster.
+func (c *controller) applyOperatorManifests(operators []models.MonitoredOperator, manifests []Manifest, kubeconfigName string) bool {
+
 	readyOperators, _, err := c.getReadyOperators(operators)
 	if err != nil {
-		c.log.WithError(err).Errorf("Failed to fetch operators from assisted-service")
+		c.log.WithError(err).Errorf("Failed to fetch operators from cluster")
+		return false
+	}
+	if len(operators) != len(readyOperators) {
+		c.log.Infof("Not all operators are currently ready")
 		return false
 	}
 
@@ -695,6 +785,7 @@ func (c *controller) applyPostInstallManifests(operators []models.MonitoredOpera
 			}
 			return false
 		}() {
+			c.log.Infof("Skipping operator %s as its not initialized by CSV", manifest.Name)
 			continue
 		}
 

src/assisted_installer_controller/assisted_installer_controller_test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/openshift/assisted-installer/src/k8s_client"
 	"github.com/openshift/assisted-installer/src/ops"
 	"github.com/openshift/assisted-service/models"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func TestValidator(t *testing.T) {
@@ -106,6 +107,15 @@ var (
 	testIngressConfigMap = map[string]string{
 		"ca-bundle.crt": "CA",
 	}
+
+	testOperatorManifests = map[string]string{
+		"kube-descheduler": "LS0tCmFwaVZlcnNpb246IG9wZXJhdG9yLm9wZW5zaGlmdC5pby92MQpraW5kOiBLdWJlRGVzY2hlZHVsZXIKbWV0YWRhdGE6CiAgbmFtZTogY2x1c3RlcgogIG5hbWVzcGFjZTogb3BlbnNoaWZ0LWt1YmUtZGVzY2hlZHVsZXItb3BlcmF0b3IKc3BlYzoKICBsb2dMZXZlbDogTm9ybWFsCiAgbW9kZTogUHJlZGljdGl2ZQogIG9wZXJhdG9yTG9nTGV2ZWw6IE5vcm1hbAogIHByb2ZpbGVzOgogICAgLSBBZmZpbml0eUFuZFRhaW50cwogIGRlc2NoZWR1bGluZ0ludGVydmFsU2Vjb25kczogMzYwMAogIG1hbmFnZW1lbnRTdGF0ZTogTWFuYWdlZAo=",
+		"nmstate":          "CmFwaVZlcnNpb246IG5tc3RhdGUuaW8vdjEKa2luZDogTk1TdGF0ZQptZXRhZGF0YToKICBuYW1lOiBubXN0YXRlCg==",
+	}
+
+	testMonitoredOperators = []models.MonitoredOperator{
+		{Name: "kube-descheduler", Namespace: "openshift-kube-descheduler-operator", OperatorType: "olm", SubscriptionName: "cluster-kube-descheduler-operator"},
+		{Name: "nmstate", Namespace: "openshift-nmstate", OperatorType: "olm", SubscriptionName: "kubernetes-nmstate-operator"}}
 )
 
 var _ = Describe("installer HostRoleMaster role", func() {
@@ -494,8 +504,7 @@ var _ = Describe("installer HostRoleMaster role", func() {
 			mockk8sclient.EXPECT().ListNodes().Return(nodes, nil).Times(1)
 			updateProgressSuccess(done, hosts)
 			configuringSuccess()
-			for name, v := range inventoryNamesIds {
-				value := v
+			for name, value := range inventoryNamesIds {
 				mockRebootsNotifier.EXPECT().Start(gomock.Any(), name, value.Host.ID, &value.Host.InfraEnvID, gomock.Any()).Times(1)
 			}
 			exit := assistedController.waitAndUpdateNodesStatus(false)
@@ -1436,6 +1445,58 @@ var _ = Describe("installer HostRoleMaster role", func() {
 		})
 	})
 
+	Context("PostInstallConfigsK8sClient", func() {
+		Context("operator manifests", func() {
+			BeforeEach(func() {
+				GeneralWaitInterval = 1 * time.Millisecond
+			})
+
+			It("success", func() {
+				mockAllCapabilitiesEnabled()
+
+				operatorConfigMap := createConfigMap()
+
+				mockk8sclient.EXPECT().GetConfigMap("assisted-installer", "olm-operator-manifests").Return(operatorConfigMap, nil).Times(1)
+				mockk8sclient.EXPECT().ListJobs(gomock.Any()).Return(&batchV1.JobList{}, nil).AnyTimes()
+				mockk8sclient.EXPECT().GetAllInstallPlansOfSubscription(gomock.Any()).Return([]olmv1alpha1.InstallPlan{}, nil).Times(2)
+				mockRebootsNotifier.EXPECT().GetKubeconfigPath(gomock.Any()).Return("", nil).Times(1)
+				randomCSV := uuid.New().String()
+				mockk8sclient.EXPECT().GetCSVFromSubscription("openshift-kube-descheduler-operator", "cluster-kube-descheduler-operator").Return(randomCSV, nil).Times(1)
+				mockk8sclient.EXPECT().GetCSV("openshift-kube-descheduler-operator", gomock.Any()).Return(&olmv1alpha1.ClusterServiceVersion{Status: olmv1alpha1.ClusterServiceVersionStatus{Phase: olmv1alpha1.CSVPhaseNone}}, nil).Times(1)
+				mockk8sclient.EXPECT().GetCSVFromSubscription("openshift-nmstate", "kubernetes-nmstate-operator").Return(randomCSV, nil).Times(1)
+				mockk8sclient.EXPECT().GetCSV("openshift-nmstate", gomock.Any()).Return(&olmv1alpha1.ClusterServiceVersion{Status: olmv1alpha1.ClusterServiceVersionStatus{Phase: olmv1alpha1.CSVPhaseNone}}, nil).Times(1)
+				mockops.EXPECT().CreateManifests(gomock.Any(), gomock.Any()).Times(2)
+
+				wg.Add(1)
+				assistedController.PostInstallConfigsK8sClient(context.TODO(), &wg, "")
+				wg.Wait()
+			})
+
+			It("failure on CSVs", func() {
+				GeneralProgressUpdateInt = 1 * time.Millisecond
+				mockAllCapabilitiesEnabled()
+
+				operatorConfigMap := createConfigMap()
+
+				mockk8sclient.EXPECT().GetConfigMap("assisted-installer", "olm-operator-manifests").Return(operatorConfigMap, nil).AnyTimes()
+				mockk8sclient.EXPECT().ListJobs(gomock.Any()).Return(&batchV1.JobList{}, nil).AnyTimes()
+				mockk8sclient.EXPECT().GetAllInstallPlansOfSubscription(gomock.Any()).Return([]olmv1alpha1.InstallPlan{}, nil).AnyTimes()
+				mockRebootsNotifier.EXPECT().GetKubeconfigPath(gomock.Any()).Return("", nil).AnyTimes()
+				mockk8sclient.EXPECT().GetCSVFromSubscription("openshift-kube-descheduler-operator", "cluster-kube-descheduler-operator").Return("", fmt.Errorf("dummy")).AnyTimes()
+				mockk8sclient.EXPECT().GetCSVFromSubscription("openshift-nmstate", "kubernetes-nmstate-operator").Return("", fmt.Errorf("dummy")).AnyTimes()
+
+				wg.Add(1)
+				go func() {
+					defer GinkgoRecover()
+					ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Millisecond)
+					defer cancel()
+					assistedController.PostInstallConfigsK8sClient(ctx, &wg, "")
+				}()
+				wg.Wait()
+			})
+		})
+	})
+
 	Context("update BMHs", func() {
 		t := metav1.Unix(98754, 0)
 		bmhStatus := metal3v1alpha1.BareMetalHostStatus{
@@ -2252,3 +2313,45 @@ func create3Hosts(currentStatus string, stage models.HostStage, nodeLabels strin
 		"node1": {Host: &models.Host{InfraEnvID: infraEnvId, ID: &node1Id, NodeLabels: nodeLabels, Progress: &currentState, Status: &currentStatus}},
 		"node2": {Host: &models.Host{InfraEnvID: infraEnvId, ID: &node2Id, NodeLabels: nodeLabels, Progress: &currentState, Status: &currentStatus}}}
 }
+
+func createConfigMap() *v1.ConfigMap {
+	type operatorMetadata struct {
+		Namespace        string   `yaml:"namespace"`
+		SubscriptionName string   `yaml:"subscriptionName"`
+		Manifests        []string `yaml:"manifests"`
+	}
+
+	configMap := v1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "olm-operator-manifests",
+			Namespace: "assisted-installer",
+		},
+		Data: map[string]string{},
+	}
+
+	for _, operator := range testMonitoredOperators {
+		var manifestFileNames []string
+		manifestContent, _ := testOperatorManifests[operator.Name]
+
+		fileName := fmt.Sprintf("%s-1.yaml", operator.Name)
+		configMap.Data[fileName] = manifestContent
+		manifestFileNames = append(manifestFileNames, fileName)
+
+		metadata := operatorMetadata{
+			Namespace:        operator.Namespace,
+			SubscriptionName: operator.SubscriptionName,
+			Manifests:        manifestFileNames,
+		}
+
+		metadataYAML, _ := yaml.Marshal(metadata)
+
+		metadataKey := fmt.Sprintf("%s.metadata.yaml", operator.Name)
+		configMap.Data[metadataKey] = string(metadataYAML)
+	}
+
+	return &configMap
+}

src/assisted_installer_controller/mock_controller.go

@@ -30,6 +30,7 @@ const (
 type RebootsNotifier interface {
 	Start(ctx context.Context, nodeName string, hostId, infraenvId, clusterId *strfmt.UUID)
 	Finalize()
+	GetKubeconfigPath(ctx context.Context) (string, error)
 }
 
 type rebootsNotifier struct {
@@ -52,7 +53,7 @@ func NewRebootsNotifier(ops ops.Ops, ic inventory_client.InventoryClient, enable
 	}
 }
 
-func (r *rebootsNotifier) getKubeconfigPath(ctx context.Context) (string, error) {
+func (r *rebootsNotifier) GetKubeconfigPath(ctx context.Context) (string, error) {
 	if r.kubeconfigPath != "" {
 		return r.kubeconfigPath, nil
 	}
@@ -75,7 +76,7 @@ func (r *rebootsNotifier) getKubeconfigPath(ctx context.Context) (string, error)
 
 func (r *rebootsNotifier) run(ctx context.Context, nodeName string, hostId, infraenvId, clusterId *strfmt.UUID) {
 	defer r.wg.Done()
-	kubeconfigPath, err := r.getKubeconfigPath(ctx)
+	kubeconfigPath, err := r.GetKubeconfigPath(ctx)
 	if err != nil {
 		r.log.Warningf("failed to get kubeconfig.  aborting notifying reboots for %s", nodeName)
 		return

src/assisted_installer_controller/mock_reboots_notifier.go

@@ -48,13 +48,13 @@ var _ = Describe("Reboots notifier", func() {
 		}
 		mockclient.EXPECT().DownloadClusterCredentials(context.TODO(), gomock.Any(), gomock.Any()).Return(errors.New("error"))
 		mockclient.EXPECT().DownloadClusterCredentials(context.TODO(), gomock.Any(), gomock.Any()).Return(nil)
-		kc, err := notifierImpl.getKubeconfigPath(context.TODO())
+		kc, err := notifierImpl.GetKubeconfigPath(context.TODO())
 		Expect(err).To(HaveOccurred())
 		Expect(kc).To(BeEmpty())
-		kc, err = notifierImpl.getKubeconfigPath(context.TODO())
+		kc, err = notifierImpl.GetKubeconfigPath(context.TODO())
 		Expect(err).ToNot(HaveOccurred())
 		Expect(kc).ToNot(BeEmpty())
-		kc2, err := notifierImpl.getKubeconfigPath(context.TODO())
+		kc2, err := notifierImpl.GetKubeconfigPath(context.TODO())
 		Expect(err).ToNot(HaveOccurred())
 		Expect(kc).To(Equal(kc2))
 	})