test: Port tests to XKS

.github/workflows/kind-ci-automation.yaml

@@ -63,6 +63,13 @@ jobs:
         run: |
           make install
 
+      - name: Install Argo CD cli tool
+        run: |
+          curl -fsSL -o /tmp/argocd-linux-amd64 https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
+          sudo install -m 555 /tmp/argocd-linux-amd64 /usr/local/bin/argocd
+          rm /tmp/argocd-linux-amd64
+          argocd version --client
+
       - name: Deploy operator
         run: |
           set -o pipefail

test/examples/operator-acceptance/argocd.yaml

@@ -1,4 +1,4 @@
-apiVersion: argoproj.io/v1alpha1
+apiVersion: argoproj.io/v1beta1
 kind: ArgoCD
 metadata:
   name: argocd

test/openshift/e2e/ginkgo/fixture/argocd/fixture.go

@@ -1,8 +1,10 @@
 package argocd
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"os/exec"
 	"strings"
 	"time"
@@ -44,6 +46,20 @@ func Update(obj *argov1beta1api.ArgoCD, modify func(*argov1beta1api.ArgoCD)) {
 	time.Sleep(7 * time.Second)
 }
 
+// CreateNewArgoCDInstance creates a new ArgoCD instance with an empty (zero) spec in the
+// given namespace and returns it. Callers should wait for availability via BeAvailable.
+func CreateNewArgoCDInstance(name, namespace string) *argov1beta1api.ArgoCD {
+	k8sClient, _ := utils.GetE2ETestKubeClient()
+
+	argoCD := &argov1beta1api.ArgoCD{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
+		Spec:       argov1beta1api.ArgoCDSpec{},
+	}
+	Expect(k8sClient.Create(context.Background(), argoCD)).To(Succeed())
+
+	return argoCD
+}
+
 func GetOpenShiftGitOpsNSArgoCD() (*argov1beta1api.ArgoCD, error) {
 
 	k8sClient, _ := utils.GetE2ETestKubeClient()
@@ -299,6 +315,96 @@ func LogInToDefaultArgoCDInstance() error {
 
 }
 
+// LogInToArgoCDInstanceWithoutRoute logs in to an ArgoCD instance via kubectl
+// port-forward instead of an OpenShift Route, so it works on xks clusters.
+// instanceName is the ArgoCD CR name (e.g. "openshift-gitops"); namespace is its namespace.
+// The returned cancel func stops the port-forward; call it (or defer it) after all argocd
+// CLI calls in the test are done, since the CLI stores localhost:18080 as the server address.
+func LogInToArgoCDInstanceWithoutRoute(instanceName, namespace string) (func(), error) {
+	k8sClient, _, err := utils.GetE2ETestKubeClientWithError()
+	if err != nil {
+		return nil, err
+	}
+
+	secretName := instanceName + "-cluster"
+	secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}}
+	if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(secret), secret); err != nil {
+		return nil, fmt.Errorf("unable to locate %q Secret", secretName)
+	}
+
+	const localPort = "18080"
+	cancel := portForwardArgoCD(namespace, "svc/"+instanceName+"-server", localPort+":80")
+
+	output, err := RunArgoCDCLI("login", "localhost:"+localPort, "--username", "admin",
+		"--password", string(secret.Data["admin.password"]), "--insecure")
+	if err != nil {
+		cancel()
+		return nil, err
+	}
+
+	if !strings.Contains(output, "'admin:login' logged in successfully") {
+		cancel()
+		return nil, fmt.Errorf("unable to log in to ArgoCD instance %q in namespace %q", instanceName, namespace)
+	}
+
+	return cancel, nil
+}
+
+// portForwardArgoCD starts kubectl port-forward and returns a cancel func.
+// Blocks until the tunnel is ready (or Fail()s after 60s).
+func portForwardArgoCD(namespace, subject, port string) func() {
+	// Kill any stale process on the local port left by a previous crashed run.
+	localPort := strings.SplitN(port, ":", 2)[0]
+	// #nosec G204
+	_ = exec.Command("sh", "-c", "lsof -ti :"+localPort+" | xargs kill -9 2>/dev/null; fuser -k "+localPort+"/tcp 2>/dev/null; true").Run()
+
+	cmd := exec.Command("kubectl", "port-forward", "-n", namespace, subject, port) // #nosec G204
+
+	stdout, err := cmd.StdoutPipe()
+	Expect(err).ToNot(HaveOccurred())
+	stderr, err := cmd.StderrPipe()
+	Expect(err).ToNot(HaveOccurred())
+
+	ready := make(chan struct{})
+
+	stream := func(r io.Reader, signal func()) {
+		defer GinkgoRecover()
+		sc := bufio.NewScanner(r)
+		for sc.Scan() {
+			line := sc.Text()
+			GinkgoWriter.Println("port-forward:", line)
+			if signal != nil && strings.HasPrefix(line, "Forwarding from") {
+				signal()
+				signal = nil
+			}
+		}
+	}
+
+	Expect(cmd.Start()).To(Succeed())
+	go stream(stdout, func() { close(ready) })
+	go stream(stderr, nil)
+	go func() {
+		defer GinkgoRecover()
+		if waitErr := cmd.Wait(); waitErr != nil &&
+			!strings.Contains(waitErr.Error(), "killed") &&
+			!strings.Contains(waitErr.Error(), "signal: killed") {
+			GinkgoWriter.Println("port-forward exited:", waitErr)
+		}
+	}()
+
+	select {
+	case <-ready:
+	case <-time.After(60 * time.Second):
+		Fail("timed out waiting for port-forward to be ready")
+	}
+
+	return func() {
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+		}
+	}
+}
+
 func RunArgoCDCLI(args ...string) (string, error) {
 
 	cmdArgs := append([]string{"argocd"}, args...)

test/openshift/e2e/ginkgo/fixture/fixture.go

@@ -273,6 +273,16 @@ func CreateRandomE2ETestNamespaceWithCleanupFunc() (*corev1.Namespace, func()) {
 	return ns, nsDeletionFunc(ns)
 }
 
+// CreateNamespaceWithArgoCDInstance creates a random namespace, creates an ArgoCD instance with
+// the given name in it, waits for it to be available, and returns the ArgoCD, namespace, and a
+// cleanup func that deletes the namespace.
+func CreateNamespaceWithArgoCDInstance(instanceName string) (*argov1beta1api.ArgoCD, *corev1.Namespace, func()) {
+	ns, cleanupFunc := CreateRandomE2ETestNamespaceWithCleanupFunc()
+	argoCDInstance := argocd.CreateNewArgoCDInstance(instanceName, ns.Name)
+	Eventually(argoCDInstance, "5m", "5s").Should(argocd.BeAvailable())
+	return argoCDInstance, ns, cleanupFunc
+}
+
 // Create namespace for tests having a specific label for identification
 // - If the namespace already exists, it will be deleted first
 func CreateNamespace(name string) *corev1.Namespace {
@@ -670,7 +680,7 @@ func WaitForAllDeploymentsInTheNamespaceToBeReady(ns string, k8sClient client.Cl
 		// All Deployments in NS are reconciled and ready
 		return true
 
-	}, "3m", "1s").Should(BeTrue())
+	}, "5m", "1s").Should(BeTrue())
 
 	// The above logic will successfully wait for Deployments to be ready. However, this does not mean that the operator's controller logic has completed it's initial cluster reconciliation logic (starting a watch then reconciling existing resources)
 	// - I'm not aware of a way to detect when this has completed, so instead I am inserting a 15 second pause.

test/openshift/e2e/ginkgo/sequential/1-006_validate_machine_config_test.go

@@ -33,20 +33,22 @@ import (
 	argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
 	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 	Context("1-006_validate_machine_config", func() {
-		// TODO: check if this test can use a new ArgoCD instance instead of the default openshift-gitops instance
 
 		var (
-			ctx           context.Context
-			k8sClient     client.Client
-			defaultArgoCD *argov1beta1api.ArgoCD
-			app           *argocdv1alpha1.Application
+			ctx         context.Context
+			k8sClient   client.Client
+			ns          *corev1.Namespace
+			cleanupFunc func()
+			argoCD      *argov1beta1api.ArgoCD
+			app         *argocdv1alpha1.Application
 		)
 
 		BeforeEach(func() {
@@ -57,43 +59,55 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 		AfterEach(func() {
 
-			fixture.OutputDebugOnFail("openshift-gitops")
+			if ns != nil {
+				fixture.OutputDebugOnFail(ns.Name)
+			}
 
-			if defaultArgoCD != nil {
+			// Restore the operator Subscription/Deployment to remove the cluster-config namespace we added
+			fixture.RestoreSubcriptionToDefault()
 
-				argocdFixture.Update(defaultArgoCD, func(ac *argov1beta1api.ArgoCD) {
-					ac.Spec.Repo.Replicas = nil
-				})
+			if cleanupFunc != nil {
+				cleanupFunc()
 			}
+		})
 
-			if app != nil {
-				Expect(k8sClient.Delete(ctx, app)).To(Succeed())
+		It("verifies that repo server replicas can be modified via .spec.repo.replicas", func() {
+
+			// The Application in this test deploys a cluster-scoped resource (config.openshift.io/v1 Image),
+			// so the Argo CD instance must be cluster-scoped. That requires setting an env var on the operator,
+			// which is not possible when the operator runs locally.
+			if fixture.EnvLocalRun() {
+				Skip("Skipping test as LOCAL_RUN env var is set. In this case, it is not possible to set env var on gitops operator controller process.")
+				return
 			}
-		})
 
-		It("verifies that repo server replicas can be modified via .spec.repo.replicas", Label("openshift"), func() {
+			By("creating a new namespace for the Argo CD instance")
+			ns, cleanupFunc = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
-			By("setting the repo server replicas to 2 on openshift-gitops Argo CD")
-			var err error
-			defaultArgoCD, err = argocdFixture.GetOpenShiftGitOpsNSArgoCD()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(defaultArgoCD).ToNot(BeNil())
+			By("adding the new namespace to ARGOCD_CLUSTER_CONFIG_NAMESPACES so the instance is cluster-scoped")
+			fixture.SetEnvInOperatorSubscriptionOrDeployment("ARGOCD_CLUSTER_CONFIG_NAMESPACES", "openshift-gitops, "+ns.Name)
 
-			argocdFixture.Update(defaultArgoCD, func(ac *argov1beta1api.ArgoCD) {
-				ac.Spec.Repo.Replicas = new(int32(2))
+			By("creating a new Argo CD instance within the namespace")
+			argoCD = argocdFixture.CreateNewArgoCDInstance("argocd", ns.Name)
+			Eventually(argoCD, "8m", "5s").Should(argocdFixture.BeAvailable())
+
+			By("setting the repo server replicas to 2 on the Argo CD instance")
+			argocdFixture.Update(argoCD, func(ac *argov1beta1api.ArgoCD) {
+				replicas := int32(2)
+				ac.Spec.Repo.Replicas = &replicas
 			})
 
 			By("creating an Argo CD Application targeting the Argo CD namespace")
 			app = &argocdv1alpha1.Application{
-				ObjectMeta: metav1.ObjectMeta{Name: "validate-machine-config", Namespace: defaultArgoCD.Namespace},
+				ObjectMeta: metav1.ObjectMeta{Name: "validate-machine-config", Namespace: ns.Name},
 				Spec: argocdv1alpha1.ApplicationSpec{
 					Source: &argocdv1alpha1.ApplicationSource{
-						Path:           "./test/examples/image",
+						Path:           "./test/examples/nginx",
 						RepoURL:        "https://github.com/redhat-developer/gitops-operator",
 						TargetRevision: "HEAD",
 					},
 					Destination: argocdv1alpha1.ApplicationDestination{
-						Namespace: defaultArgoCD.Namespace,
+						Namespace: ns.Name,
 						Server:    "https://kubernetes.default.svc",
 					},
 					Project: "default",
@@ -108,27 +122,26 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			Expect(k8sClient.Create(ctx, app)).To(Succeed())
 
 			By("waiting for Argo CD to become available after the repo server change we made")
-			Eventually(defaultArgoCD, "5m", "5s").Should(argocdFixture.BeAvailable())
+			Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable())
 
 			By("verifying deployment and statefulset have expected number of replicas, including the repo server which should have 2")
 			deploymentsToVerify := []string{
-				"openshift-gitops-server",
-				"openshift-gitops-redis",
-				"openshift-gitops-applicationset-controller",
-				"openshift-gitops-repo-server",
+				"argocd-server",
+				"argocd-redis",
+				"argocd-repo-server",
 			}
 
 			for _, deplToVerify := range deploymentsToVerify {
 
 				depl := &appsv1.Deployment{
-					ObjectMeta: metav1.ObjectMeta{Name: deplToVerify, Namespace: defaultArgoCD.Namespace},
+					ObjectMeta: metav1.ObjectMeta{Name: deplToVerify, Namespace: ns.Name},
 				}
 				Eventually(depl).Should(k8sFixture.ExistByName())
 
 				expectedReadyReplicas := 1
 				expectedReplicas := 1
 
-				if deplToVerify == "openshift-gitops-repo-server" {
+				if deplToVerify == "argocd-repo-server" {
 					expectedReadyReplicas = 2
 					expectedReplicas = 2
 				}
@@ -138,8 +151,8 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 			ss := &appsv1.StatefulSet{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      "openshift-gitops-application-controller",
-					Namespace: defaultArgoCD.Namespace,
+					Name:      "argocd-application-controller",
+					Namespace: ns.Name,
 				},
 			}
 			Eventually(ss).Should(k8sFixture.ExistByName())
@@ -151,13 +164,17 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			Eventually(app, "4m", "5s").Should(application.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
 
 			By("updating repo server replicas back to 1")
-			argocdFixture.Update(defaultArgoCD, func(ac *argov1beta1api.ArgoCD) {
-				ac.Spec.Repo.Replicas = new(int32(1))
+			argocdFixture.Update(argoCD, func(ac *argov1beta1api.ArgoCD) {
+				replicas := int32(1)
+				ac.Spec.Repo.Replicas = &replicas
 			})
 
+			By("waiting for Argo CD to become available after the repo server change we made")
+			Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable())
+
 			By("verifying repo server Deployment moves back to a single replica")
 			repoServerDepl := &appsv1.Deployment{
-				ObjectMeta: metav1.ObjectMeta{Name: "openshift-gitops-repo-server", Namespace: defaultArgoCD.Namespace},
+				ObjectMeta: metav1.ObjectMeta{Name: "argocd-repo-server", Namespace: ns.Name},
 			}
 			Eventually(repoServerDepl).Should(k8sFixture.ExistByName())
 			Eventually(repoServerDepl).Should(deployment.HaveReplicas(1))