chore: add test fixes and additional debugging

test/openshift/e2e/ginkgo/fixture/fixture.go

@@ -869,6 +869,12 @@ func OutputDebug(namespaceParams ...any) {
 			namespaces = append(namespaces, str)
 
 		} else if nsPtr, isNsPtr := (param).(*corev1.Namespace); isNsPtr {
+			// A typed nil (*corev1.Namespace)(nil) is not caught by the 'param == nil'
+			// check above, since an interface holding a typed nil is itself non-nil.
+			// Skip it here to avoid dereferencing a nil pointer.
+			if nsPtr == nil {
+				continue
+			}
 			namespaces = append(namespaces, nsPtr.Name)
 
 		} else if ns, isNs := (param).(corev1.Namespace); isNs {

test/openshift/e2e/ginkgo/parallel/1-021_validate_rolebindings_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
 	argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
 	fixtureUtils "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
+	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -36,8 +37,10 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 	Context("1-021_validate_rolebindings", func() {
 
 		var (
-			ctx       context.Context
-			k8sClient client.Client
+			ctx         context.Context
+			k8sClient   client.Client
+			randomNS    *corev1.Namespace
+			cleanupFunc func()
 		)
 
 		BeforeEach(func() {
@@ -46,11 +49,17 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			ctx = context.Background()
 		})
 
+		AfterEach(func() {
+			fixture.OutputDebugOnFail(randomNS)
+			if cleanupFunc != nil {
+				cleanupFunc()
+			}
+		})
+
 		It("verifies a namespace-scoped Argo CD instance has the expected RoleBindings", func() {
 
 			By("creating new namespace-scoped Argo CD instance and verifying it becomes available")
-			randomNS, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			randomNS, cleanupFunc = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			argoCD := &argov1beta1api.ArgoCD{
 				ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: randomNS.Name},

test/openshift/e2e/ginkgo/parallel/1-025-validate-managed-by-change_test.go

@@ -126,7 +126,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo2.Name, k8sClient)
 
 			By("verifying Argo CD Application deployed as expected and is healthy and synced")
-			Eventually(app, "3m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
+			Eventually(app, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
 			Eventually(app, "60s", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
 
 			By("update 'test_1_25_target' NS to be managed by the second Argo CD instance, rather than the first")

test/openshift/e2e/ginkgo/parallel/1-029_validate_tls_secret_no_scale_test.go

@@ -44,6 +44,8 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 		var (
 			ctx       context.Context
 			k8sClient client.Client
+			argoCDNS  *corev1.Namespace
+			cleanup1  func()
 		)
 
 		BeforeEach(func() {
@@ -52,11 +54,17 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			ctx = context.Background()
 		})
 
+		AfterEach(func() {
+			fixture.OutputDebugOnFail(argoCDNS)
+			if cleanup1 != nil {
+				cleanup1()
+			}
+		})
+
 		It("ensures that Argo CD server becomes ready after modifying Argo CD Server TLS secret", func() {
 
 			By("creating basic Argo CD instance")
-			argoCDNS, cleanup1 := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanup1()
+			argoCDNS, cleanup1 = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			argoCD := &argov1beta1api.ArgoCD{
 				ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: argoCDNS.Name},

test/openshift/e2e/ginkgo/parallel/1-030_validate_reencrypt_test.go

@@ -44,8 +44,10 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 	Context("1-030_validate_reencrypt", func() {
 
 		var (
-			ctx       context.Context
-			k8sClient client.Client
+			ctx             context.Context
+			k8sClient       client.Client
+			test_1_30_argo1 *corev1.Namespace
+			cleanupFunc     func()
 		)
 
 		BeforeEach(func() {
@@ -54,12 +56,21 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			ctx = context.Background()
 		})
 
+		AfterEach(func() {
+
+			fixture.OutputDebugOnFail(test_1_30_argo1)
+
+			if cleanupFunc != nil {
+				cleanupFunc()
+			}
+
+		})
+
 		It("verifies Argo CD Server's Route can be enabled with TLSTerminationReencrypt", Label("openshift"), func() {
 
 			By("creating namespace-scoped Argo CD instance with rencrypt Route")
 
-			test_1_30_argo1, cleanupFunc := fixture.CreateNamespaceWithCleanupFunc("test-1-30-argo1")
-			defer cleanupFunc()
+			test_1_30_argo1, cleanupFunc = fixture.CreateNamespaceWithCleanupFunc("test-1-30-argo1")
 
 			argoCD := &argov1beta1api.ArgoCD{
 				ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_30_argo1.Name},

test/openshift/e2e/ginkgo/parallel/1-067_validate_redis_secure_comm_no_autotls_ha_test.go

@@ -18,7 +18,10 @@ package parallel
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"regexp"
+	"strings"
 
 	argov1beta1api "github.com/argoproj-labs/argocd-operator/api/v1beta1"
 	"github.com/argoproj-labs/argocd-operator/controllers/argoutil"
@@ -160,8 +163,6 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			expectComponentsAreRunning()
 
 			By("extracting the contents of /data/conf/redis.conf and checking it contains expected values")
-			redisConf, err := osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "exec", "-i", "pod/argocd-redis-ha-server-0", "-n", ns.Name, "-c", "redis", "--", "cat", "/data/conf/redis.conf")
-			Expect(err).ToNot(HaveOccurred())
 			expectedRedisConfig := []string{
 				"port 0",
 				"tls-port 6379",
@@ -171,17 +172,31 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 				"tls-replication yes",
 				"tls-auth-clients no",
 			}
-			for _, line := range expectedRedisConfig {
-				Expect(redisConf).To(ContainSubstring(line))
+
+			// redisConfHasExpectedValues extracts the contents of /data/conf/redis.conf and
+			// returns an error if any of the expected values are missing.
+			redisConfHasExpectedValues := func() error {
+				redisConf, err := osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "exec", "-i", "pod/argocd-redis-ha-server-0", "-n", ns.Name, "-c", "redis", "--", "cat", "/data/conf/redis.conf")
+				if err != nil {
+					return err
+				}
+				GinkgoWriter.Println("redis conf values:")
+				GinkgoWriter.Println(redisConf)
+
+				for _, line := range expectedRedisConfig {
+					if !strings.Contains(redisConf, line) {
+						return fmt.Errorf("redis.conf does not contain expected value: %s", line)
+					}
+				}
+				return nil
 			}
 
+			// First, wait for redis.conf to eventually contain the expected values, then
+			// verify it consistently contains them.
+			Eventually(redisConfHasExpectedValues, "2m", "5s").Should(Succeed())
+			Consistently(redisConfHasExpectedValues, "30s", "5s").Should(Succeed())
+
 			By("extracting the contents of /data/conf/sentinel.conf and checking it contains expected values")
-			sentinelConf, err := osFixture.ExecCommandWithOutputParam(
-				false, true,
-				"kubectl", "exec", "-i", "pod/argocd-redis-ha-server-0", "-n", ns.Name, "-c", "redis",
-				"--", "cat", "/data/conf/sentinel.conf",
-			)
-			Expect(err).ToNot(HaveOccurred())
 			expectedSentinelConfig := []string{
 				"port 0",
 				"tls-port 26379",
@@ -192,10 +207,38 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 				"tls-replication yes",
 				"tls-auth-clients no",
 			}
-			for _, line := range expectedSentinelConfig {
-				Expect(sentinelConf).To(MatchRegexp(line))
+
+			// sentinelConfHasExpectedValues extracts the contents of /data/conf/sentinel.conf
+			// and returns an error if any of the expected values are missing.
+			sentinelConfHasExpectedValues := func() error {
+				sentinelConf, err := osFixture.ExecCommandWithOutputParam(
+					false, true,
+					"kubectl", "exec", "-i", "pod/argocd-redis-ha-server-0", "-n", ns.Name, "-c", "redis",
+					"--", "cat", "/data/conf/sentinel.conf",
+				)
+				if err != nil {
+					return err
+				}
+				GinkgoWriter.Println("sentinel conf values:")
+				GinkgoWriter.Println(sentinelConf)
+
+				for _, line := range expectedSentinelConfig {
+					matched, err := regexp.MatchString(line, sentinelConf)
+					if err != nil {
+						return err
+					}
+					if !matched {
+						return fmt.Errorf("sentinel.conf does not contain expected value: %s", line)
+					}
+				}
+				return nil
 			}
 
+			// First, wait for sentinel.conf to eventually contain the expected values, then
+			// verify it consistently contains them.
+			Eventually(sentinelConfHasExpectedValues, "2m", "5s").Should(Succeed())
+			Consistently(sentinelConfHasExpectedValues, "30s", "5s").Should(Succeed())
+
 			fqdnSuffix := ".svc.cluster.local.:"
 
 			repoServerDepl := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "argocd-repo-server", Namespace: ns.Name}}

test/openshift/e2e/ginkgo/parallel/1-132_validate_sensitive_annotation_masking_test.go

@@ -52,8 +52,12 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 		const sensitiveToken = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.fake-openshift-service-account-token"
 
 		var (
-			k8sClient client.Client
-			ctx       context.Context
+			k8sClient       client.Client
+			ctx             context.Context
+			argoCDNS        *corev1.Namespace
+			cleanupArgoCDNS func()
+			appNS           *corev1.Namespace
+			cleanupAppNS    func()
 		)
 
 		BeforeEach(func() {
@@ -62,13 +66,24 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			ctx = context.Background()
 		})
 
+		AfterEach(func() {
+
+			fixture.OutputDebugOnFail(argoCDNS, appNS)
+
+			if cleanupAppNS != nil {
+				cleanupAppNS()
+			}
+			if cleanupArgoCDNS != nil {
+				cleanupArgoCDNS()
+			}
+		})
+
 		It("verifies that resource.sensitive.mask.annotations is set in argocd-cm and that the openshift.io/token-secret.value annotation is hidden from diff computation so the app stays Synced and the token is never visible in CLI output", func() {
 
 			fixture.EnsureRunningOnOpenShift()
 
 			By("creating namespace for the ArgoCD instance")
-			argoCDNS, cleanupArgoCDNS := fixture.CreateNamespaceWithCleanupFunc("test-1-132-argocd")
-			defer cleanupArgoCDNS()
+			argoCDNS, cleanupArgoCDNS = fixture.CreateNamespaceWithCleanupFunc("test-1-132-argocd")
 
 			By("creating a namespace-scoped ArgoCD instance with the server Route enabled")
 			argoCD := &argov1beta1api.ArgoCD{
@@ -90,8 +105,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			Eventually(argocdCM).Should(configmapFixture.HaveStringDataKeyValue("resource.sensitive.mask.annotations", tokenAnnotationKey))
 
 			By("creating a managed namespace for app deployment")
-			appNS, cleanupAppNS := fixture.CreateManagedNamespaceWithCleanupFunc("test-1-132-apps", argoCDNS.Name)
-			defer cleanupAppNS()
+			appNS, cleanupAppNS = fixture.CreateManagedNamespaceWithCleanupFunc("test-1-132-apps", argoCDNS.Name)
 
 			By("creating a per-test ArgoCD CLI config file to prevent parallel-test login context conflicts")
 			cliConfigFile, err := os.CreateTemp("", "argocd-e2e-1-132-*.yaml")

test/openshift/e2e/ginkgo/sequential/1-040_validate_quoted_RBAC_group_names_test.go

@@ -5,14 +5,17 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	routev1 "github.com/openshift/api/route/v1"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
 	argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
+	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
+	routeFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/route"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 	Context("1-040-validate_quoted_RBAC_group_names", func() {
-		// TODO: check if this test can use a new ArgoCD instance instead of the default openshift-gitops instance
 
 		BeforeEach(func() {
 			fixture.EnsureSequentialCleanSlate()
@@ -37,6 +40,16 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(defaultArgoCD, "5m", "5s").Should(argocdFixture.BeAvailable())
 
+			By("verifying the argocd-server route in openshift-gitops namespace has been admitted, so avoid short race condition where Argo CD is deployed, but Route isn't available yet, so it can't be used to log in")
+			serverRoute := &routev1.Route{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "openshift-gitops-server",
+					Namespace: "openshift-gitops",
+				},
+			}
+			Eventually(serverRoute).Should(k8sFixture.ExistByName())
+			Eventually(serverRoute).Should(routeFixture.HaveAdmittedIngress())
+
 			By("logging in to Argo CD instance")
 			Expect(argocdFixture.LogInToDefaultArgoCDInstance()).To(Succeed())
 

test/openshift/e2e/ginkgo/sequential/1-054_validate_argocd_agent_destination_mapping_test.go

@@ -25,6 +25,7 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
 	appFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/application"
+	argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
 	deploymentFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
 	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
 	fixtureUtils "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
@@ -263,8 +264,9 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			}
 		}
 
-		// deployDestMapPrincipal deploys the principal ArgoCD instance with destination-based mapping.
-		deployDestMapPrincipal := func() {
+		// deployDestMapPrincipal deploys the principal ArgoCD instance with destination-based mapping,
+		// and returns the created ArgoCD CR.
+		deployDestMapPrincipal := func() *argov1beta1api.ArgoCD {
 			GinkgoHelper()
 
 			nsPrincipal, cleanup := fixture.CreateNamespaceWithCleanupFunc(destMapNsPrincipal)
@@ -328,10 +330,13 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 			By("Verify principal logs contain expected messages")
 			agentFixture.VerifyLogs(destMapDeploymentPrincipal, nsPrincipal.Name, destMapPrincipalStartupLogs)
+
+			return argoCDInstance
 		}
 
-		// deployDestMapAgent deploys the managed agent ArgoCD instance with destination-based mapping.
-		deployDestMapAgent := func() {
+		// deployDestMapAgent deploys the managed agent ArgoCD instance with destination-based mapping,
+		// and returns the created ArgoCD CR.
+		deployDestMapAgent := func() *argov1beta1api.ArgoCD {
 			GinkgoHelper()
 
 			nsAgent, cleanup := fixture.CreateNamespaceWithCleanupFunc(destMapNsAgent)
@@ -370,6 +375,8 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 			By("Verify managed agent logs contain expected messages")
 			agentFixture.VerifyLogs(destMapDeploymentAgent, nsAgent.Name, destMapAgentStartupLogs)
+
+			return argoCDInstanceAgent
 		}
 
 		createAdminCRBForAgent := func() {
@@ -447,10 +454,14 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		It("Should deploy principal and agent with destination-based mapping", Label("openshift"), func() {
 
 			By("Deploy principal with destination-based mapping enabled")
-			deployDestMapPrincipal()
+			principalArgoCD := deployDestMapPrincipal()
 
 			By("Deploy managed agent with destination-based mapping enabled")
-			deployDestMapAgent()
+			agentArgoCD := deployDestMapAgent()
+
+			By("Verify principal and agent ArgoCD instances are available")
+			Eventually(principalArgoCD, "5m", "5s").Should(argocdFixture.BeAvailable())
+			Eventually(agentArgoCD, "5m", "5s").Should(argocdFixture.BeAvailable())
 
 			By("Verify principal is connected to the managed agent")
 			agentFixture.VerifyLogs(destMapDeploymentPrincipal, destMapNsPrincipal, []string{
@@ -480,9 +491,9 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 				"Application should be routed from principal to agent namespace")
 
 			By("Verify application on agent is synced and healthy")
-			Eventually(agentApp, "120s", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced),
+			Eventually(agentApp, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced),
 				"Application on agent should be synced")
-			Eventually(agentApp, "120s", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy),
+			Eventually(agentApp, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy),
 				"Application on agent should be healthy")
 
 			By("Verify application is synced and healthy on the principal")