Add tests cases for upgrade

controllers/bounce_processes.go

@@ -265,6 +265,7 @@ func getProcessesReadyForRestart(logger logr.Logger, cluster *fdbv1beta2.Foundat
 
 	if cluster.Spec.LogServersPerPod > 1 {
 		expectedProcesses += counts.Log * (cluster.Spec.LogServersPerPod - 1)
+		expectedProcesses += counts.Transaction * (cluster.Spec.LogServersPerPod - 1)
 	}
 
 	if cluster.IsBeingUpgradedWithVersionIncompatibleVersion() && expectedProcesses != len(addresses) {

controllers/cluster_controller_test.go

@@ -3160,7 +3160,7 @@ var _ = Describe("cluster_controller", func() {
 				Expect(err).NotTo(HaveOccurred())
 			})
 
-			It("should generate the storage conf with two processes", func() {
+			It("should generate the log conf with two processes", func() {
 				Expect(conf).To(Equal(strings.Join([]string{
 					"[general]",
 					"kill_on_configuration_change = false",

e2e/fixtures/fdb_cluster.go

@@ -504,6 +504,11 @@ func (fdbCluster *FdbCluster) GetStoragePods() *corev1.PodList {
 	return fdbCluster.getPodsByProcessClass(fdbv1beta2.ProcessClassStorage)
 }
 
+// GetTransactionPods returns all Pods of this cluster that have the process class transaction.
+func (fdbCluster *FdbCluster) GetTransactionPods() *corev1.PodList {
+	return fdbCluster.getPodsByProcessClass(fdbv1beta2.ProcessClassTransaction)
+}
+
 // GetPod returns the Pod with the given name that runs in the same namespace as the FoundationDBCluster.
 func (fdbCluster *FdbCluster) GetPod(name string) *corev1.Pod {
 	pod := &corev1.Pod{}
@@ -596,6 +601,27 @@ func (fdbCluster *FdbCluster) SetStorageServerPerPod(serverPerPod int) error {
 	return fdbCluster.setStorageServerPerPod(serverPerPod, true)
 }
 
+func (fdbCluster *FdbCluster) setTransactionServerPerPod(
+	serverPerPod int,
+	processCount int,
+	waitForReconcile bool,
+) error {
+	fdbCluster.cluster.Spec.LogServersPerPod = serverPerPod
+	fdbCluster.cluster.Spec.ProcessCounts.Transaction = processCount
+	fdbCluster.cluster.Spec.ProcessCounts.Log = 0
+	fdbCluster.UpdateClusterSpec()
+
+	if !waitForReconcile {
+		return nil
+	}
+	return fdbCluster.WaitForReconciliation()
+}
+
+// SetTransactionServerPerPod set the SetTransactionServerPerPod field in the cluster spec.
+func (fdbCluster *FdbCluster) SetTransactionServerPerPod(serverPerPod int, processCount int) error {
+	return fdbCluster.setTransactionServerPerPod(serverPerPod, processCount, true)
+}
+
 // ReplacePod replaces the provided Pod if it's part of the FoundationDBCluster.
 func (fdbCluster *FdbCluster) ReplacePod(pod corev1.Pod, waitForReconcile bool) {
 	fdbCluster.cluster.Spec.ProcessGroupsToRemove = []fdbv1beta2.ProcessGroupID{GetProcessGroupID(pod)}

e2e/fixtures/status.go

@@ -307,6 +307,20 @@ func (fdbCluster *FdbCluster) GetProcessCount(targetRole fdbv1beta2.ProcessRole)
 	return pCounter
 }
 
+// GetProcessCountByProcessClass returns the number of processes based on process class
+func (fdbCluster *FdbCluster) GetProcessCountByProcessClass(pCloass fdbv1beta2.ProcessClass) int {
+	pCounter := 0
+	status := fdbCluster.GetStatus()
+
+	for _, process := range status.Cluster.Processes {
+		if process.ProcessClass == pCloass {
+			pCounter++
+		}
+	}
+
+	return pCounter
+}
+
 // HasTLSEnabled returns true if the cluster is running with TLS enabled.
 func (fdbCluster *FdbCluster) HasTLSEnabled() bool {
 	status := fdbCluster.GetStatus()

e2e/test_operator/operator_test.go

@@ -894,7 +894,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 		})
 	})
 
-	FWhen("setting 2 logs per disk", func() {
+	When("setting 2 logs per disk", func() {
 		var initialLogServerPerPod, expectedPodCnt, expectedLogProcessesCnt int
 
 		BeforeEach(func() {
@@ -907,7 +907,9 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 				expectedPodCnt,
 				expectedLogProcessesCnt,
 			)
-			validateProcessesCount(fdbCluster, fdbv1beta2.ProcessRoleLog, expectedPodCnt, expectedLogProcessesCnt)
+			Eventually(func() int {
+				return len(fdbCluster.GetLogPods().Items)
+			}).Should(BeNumerically("==", expectedPodCnt))
 		})
 
 		AfterEach(func() {
@@ -918,24 +920,76 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 				expectedPodCnt,
 				expectedPodCnt*initialLogServerPerPod,
 			)
-			validateProcessesCount(
-				fdbCluster,
-				fdbv1beta2.ProcessRoleLog,
+			Eventually(func() int {
+				return len(fdbCluster.GetLogPods().Items)
+			}).Should(BeNumerically("==", expectedPodCnt))
+		})
+
+		It("should update the log servers to the expected amount", func() {
+			serverPerPod := initialLogServerPerPod * 2
+			Expect(fdbCluster.SetLogServersPerPod(serverPerPod)).ShouldNot(HaveOccurred())
+			log.Printf(
+				"expectedPodCnt: %d, expectedStorageProcessesCnt: %d",
+				expectedPodCnt,
+				expectedPodCnt*serverPerPod,
+			)
+
+			Eventually(func() int {
+				return len(fdbCluster.GetLogPods().Items)
+			}).Should(BeNumerically("==", expectedPodCnt))
+			Eventually(func() int {
+				return fdbCluster.GetProcessCountByProcessClass(fdbv1beta2.ProcessClassLog)
+			}).Should(BeNumerically("==", 10))
+		})
+	})
+
+	FWhen("setting 2 logs per disk to use Transaction process", func() {
+		var initialLogServerPerPod, expectedPodCnt, expectedLogProcessesCnt int
+
+		BeforeEach(func() {
+			initialLogServerPerPod = fdbCluster.GetLogServersPerPod()
+			initialPods := fdbCluster.GetLogPods()
+			expectedPodCnt = len(initialPods.Items)
+			expectedLogProcessesCnt = expectedPodCnt * initialLogServerPerPod
+			log.Printf(
+				"expectedPodCnt: %d, expectedProcessesCnt: %d",
+				expectedPodCnt,
+				expectedLogProcessesCnt,
+			)
+			Eventually(func() int {
+				return len(fdbCluster.GetLogPods().Items)
+			}).Should(BeNumerically("==", expectedPodCnt))
+		})
+
+		AfterEach(func() {
+			log.Printf("set log servers per Pod to %d", initialLogServerPerPod)
+			Expect(fdbCluster.SetLogServersPerPod(initialLogServerPerPod)).ShouldNot(HaveOccurred())
+			log.Printf(
+				"expectedPodCnt: %d, expectedProcessesCnt: %d",
 				expectedPodCnt,
 				expectedPodCnt*initialLogServerPerPod,
 			)
+			Eventually(func() int {
+				return len(fdbCluster.GetLogPods().Items)
+			}).Should(BeNumerically("==", expectedPodCnt))
 		})
 
 		It("should update the log servers to the expected amount", func() {
 			serverPerPod := initialLogServerPerPod * 2
-			log.Printf("set storage server per Pod to %d", serverPerPod)
+			Expect(fdbCluster.SetTransactionServerPerPod(serverPerPod, 5)).ShouldNot(HaveOccurred())
 			Expect(fdbCluster.SetLogServersPerPod(serverPerPod)).ShouldNot(HaveOccurred())
 			log.Printf(
 				"expectedPodCnt: %d, expectedStorageProcessesCnt: %d",
 				expectedPodCnt,
 				expectedPodCnt*serverPerPod,
 			)
-			validateProcessesCount(fdbCluster, fdbv1beta2.ProcessRoleLog, expectedPodCnt, expectedPodCnt*serverPerPod)
+
+			Eventually(func() int {
+				return len(fdbCluster.GetTransactionPods().Items)
+			}).Should(BeNumerically("==", expectedPodCnt))
+			Eventually(func() int {
+				return fdbCluster.GetProcessCountByProcessClass(fdbv1beta2.ProcessClassTransaction)
+			}).Should(BeNumerically("==", 10))
 		})
 	})
 

e2e/test_operator_upgrades/operator_upgrades_test.go

@@ -1135,4 +1135,65 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 		EntryDescription("Upgrade from %[1]s to %[2]s"),
 		fixtures.GenerateUpgradeTableEntries(testOptions),
 	)
+
+	DescribeTable(
+		"with 2 log servers per Pod",
+		func(beforeVersion string, targetVersion string) {
+			clusterSetupWithConfig(beforeVersion, true, &fixtures.ClusterConfig{
+				DebugSymbols:     false,
+				LogServersPerPod: 2,
+			})
+
+			Expect(fdbCluster.UpgradeCluster(targetVersion, false)).NotTo(HaveOccurred())
+			// Make sure the cluster is still running with 2 storage server per Pod.
+			Expect(fdbCluster.GetCluster().Spec.LogServersPerPod).To(Equal(2))
+
+			if !fixtures.VersionsAreProtocolCompatible(beforeVersion, targetVersion) {
+				// Ensure that the operator is setting the IncorrectConfigMap and IncorrectCommandLine conditions during the upgrade
+				// process.
+				expectedConditions := map[fdbv1beta2.ProcessGroupConditionType]bool{
+					fdbv1beta2.IncorrectConfigMap:   true,
+					fdbv1beta2.IncorrectCommandLine: true,
+				}
+				Eventually(func() bool {
+					cluster := fdbCluster.GetCluster()
+
+					for _, processGroup := range cluster.Status.ProcessGroups {
+						if !processGroup.MatchesConditions(expectedConditions) {
+							return false
+						}
+					}
+
+					return true
+				}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(BeTrue())
+			}
+
+			transactionSystemProcessGroups := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None)
+			// Wait until the cluster is upgraded and fully reconciled.
+			Expect(fdbCluster.WaitUntilWithForceReconcile(2, 600, func(cluster *fdbv1beta2.FoundationDBCluster) bool {
+				for _, processGroup := range cluster.Status.ProcessGroups {
+					if processGroup.ProcessClass == fdbv1beta2.ProcessClassLog {
+						continue
+					}
+
+					transactionSystemProcessGroups[processGroup.ProcessGroupID] = fdbv1beta2.None{}
+				}
+
+				// Allow soft reconciliation and make sure the running version was updated
+				return cluster.Status.Generations.Reconciled == cluster.Generation && cluster.Status.RunningVersion == targetVersion
+			})).NotTo(HaveOccurred())
+
+			// Get the desired process counts based on the current cluster configuration
+			processCounts, err := fdbCluster.GetProcessCounts()
+			Expect(err).NotTo(HaveOccurred())
+
+			// During an upgrade we expect that the transaction system processes are replaced, so we expect to have seen
+			// 2 times the process counts for transaction system processes. Add a small buffer of 5 to allow automatic
+			// replacements during an upgrade.
+			expectedProcessCounts := (processCounts.Total()-processCounts.Log)*2 + 5
+			Expect(len(transactionSystemProcessGroups)).To(BeNumerically("<=", expectedProcessCounts))
+		},
+		EntryDescription("Upgrade from %[1]s to %[2]s"),
+		fixtures.GenerateUpgradeTableEntries(testOptions),
+	)
 })