From 722a020919e412687c1ac9d192e4d7f76fbc2ab6 Mon Sep 17 00:00:00 2001 From: Shambugouda annigeri Date: Tue, 4 Jul 2023 18:04:46 +0100 Subject: [PATCH] Add tests cases for upgrade --- controllers/bounce_processes.go | 1 + controllers/cluster_controller_test.go | 2 +- e2e/fixtures/fdb_cluster.go | 26 +++++++ e2e/fixtures/status.go | 14 ++++ e2e/test_operator/operator_test.go | 68 +++++++++++++++++-- .../operator_upgrades_test.go | 61 +++++++++++++++++ 6 files changed, 164 insertions(+), 8 deletions(-) diff --git a/controllers/bounce_processes.go b/controllers/bounce_processes.go index 99ccaf909..86f7f90d7 100644 --- a/controllers/bounce_processes.go +++ b/controllers/bounce_processes.go @@ -265,6 +265,7 @@ func getProcessesReadyForRestart(logger logr.Logger, cluster *fdbv1beta2.Foundat if cluster.Spec.LogServersPerPod > 1 { expectedProcesses += counts.Log * (cluster.Spec.LogServersPerPod - 1) + expectedProcesses += counts.Transaction * (cluster.Spec.LogServersPerPod - 1) } if cluster.IsBeingUpgradedWithVersionIncompatibleVersion() && expectedProcesses != len(addresses) { diff --git a/controllers/cluster_controller_test.go b/controllers/cluster_controller_test.go index 80a7254dc..98e40ee8b 100644 --- a/controllers/cluster_controller_test.go +++ b/controllers/cluster_controller_test.go @@ -3160,7 +3160,7 @@ var _ = Describe("cluster_controller", func() { Expect(err).NotTo(HaveOccurred()) }) - It("should generate the storage conf with two processes", func() { + It("should generate the log conf with two processes", func() { Expect(conf).To(Equal(strings.Join([]string{ "[general]", "kill_on_configuration_change = false", diff --git a/e2e/fixtures/fdb_cluster.go b/e2e/fixtures/fdb_cluster.go index 51ec9f65c..c74700df0 100644 --- a/e2e/fixtures/fdb_cluster.go +++ b/e2e/fixtures/fdb_cluster.go @@ -504,6 +504,11 @@ func (fdbCluster *FdbCluster) GetStoragePods() *corev1.PodList { return fdbCluster.getPodsByProcessClass(fdbv1beta2.ProcessClassStorage) } +// GetTransactionPods returns all Pods of this cluster that have the process class transaction. +func (fdbCluster *FdbCluster) GetTransactionPods() *corev1.PodList { + return fdbCluster.getPodsByProcessClass(fdbv1beta2.ProcessClassTransaction) +} + // GetPod returns the Pod with the given name that runs in the same namespace as the FoundationDBCluster. func (fdbCluster *FdbCluster) GetPod(name string) *corev1.Pod { pod := &corev1.Pod{} @@ -596,6 +601,27 @@ func (fdbCluster *FdbCluster) SetStorageServerPerPod(serverPerPod int) error { return fdbCluster.setStorageServerPerPod(serverPerPod, true) } +func (fdbCluster *FdbCluster) setTransactionServerPerPod( + serverPerPod int, + processCount int, + waitForReconcile bool, +) error { + fdbCluster.cluster.Spec.LogServersPerPod = serverPerPod + fdbCluster.cluster.Spec.ProcessCounts.Transaction = processCount + fdbCluster.cluster.Spec.ProcessCounts.Log = 0 + fdbCluster.UpdateClusterSpec() + + if !waitForReconcile { + return nil + } + return fdbCluster.WaitForReconciliation() +} + +// SetTransactionServerPerPod set the SetTransactionServerPerPod field in the cluster spec. +func (fdbCluster *FdbCluster) SetTransactionServerPerPod(serverPerPod int, processCount int) error { + return fdbCluster.setTransactionServerPerPod(serverPerPod, processCount, true) +} + // ReplacePod replaces the provided Pod if it's part of the FoundationDBCluster. func (fdbCluster *FdbCluster) ReplacePod(pod corev1.Pod, waitForReconcile bool) { fdbCluster.cluster.Spec.ProcessGroupsToRemove = []fdbv1beta2.ProcessGroupID{GetProcessGroupID(pod)} diff --git a/e2e/fixtures/status.go b/e2e/fixtures/status.go index 56589f404..f7fbba465 100644 --- a/e2e/fixtures/status.go +++ b/e2e/fixtures/status.go @@ -307,6 +307,20 @@ func (fdbCluster *FdbCluster) GetProcessCount(targetRole fdbv1beta2.ProcessRole) return pCounter } +// GetProcessCountByProcessClass returns the number of processes based on process class +func (fdbCluster *FdbCluster) GetProcessCountByProcessClass(pCloass fdbv1beta2.ProcessClass) int { + pCounter := 0 + status := fdbCluster.GetStatus() + + for _, process := range status.Cluster.Processes { + if process.ProcessClass == pCloass { + pCounter++ + } + } + + return pCounter +} + // HasTLSEnabled returns true if the cluster is running with TLS enabled. func (fdbCluster *FdbCluster) HasTLSEnabled() bool { status := fdbCluster.GetStatus() diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go index 6b1fbbc6a..01d58cbe7 100644 --- a/e2e/test_operator/operator_test.go +++ b/e2e/test_operator/operator_test.go @@ -894,7 +894,7 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() { }) }) - FWhen("setting 2 logs per disk", func() { + When("setting 2 logs per disk", func() { var initialLogServerPerPod, expectedPodCnt, expectedLogProcessesCnt int BeforeEach(func() { @@ -907,7 +907,9 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() { expectedPodCnt, expectedLogProcessesCnt, ) - validateProcessesCount(fdbCluster, fdbv1beta2.ProcessRoleLog, expectedPodCnt, expectedLogProcessesCnt) + Eventually(func() int { + return len(fdbCluster.GetLogPods().Items) + }).Should(BeNumerically("==", expectedPodCnt)) }) AfterEach(func() { @@ -918,24 +920,76 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() { expectedPodCnt, expectedPodCnt*initialLogServerPerPod, ) - validateProcessesCount( - fdbCluster, - fdbv1beta2.ProcessRoleLog, + Eventually(func() int { + return len(fdbCluster.GetLogPods().Items) + }).Should(BeNumerically("==", expectedPodCnt)) + }) + + It("should update the log servers to the expected amount", func() { + serverPerPod := initialLogServerPerPod * 2 + Expect(fdbCluster.SetLogServersPerPod(serverPerPod)).ShouldNot(HaveOccurred()) + log.Printf( + "expectedPodCnt: %d, expectedStorageProcessesCnt: %d", + expectedPodCnt, + expectedPodCnt*serverPerPod, + ) + + Eventually(func() int { + return len(fdbCluster.GetLogPods().Items) + }).Should(BeNumerically("==", expectedPodCnt)) + Eventually(func() int { + return fdbCluster.GetProcessCountByProcessClass(fdbv1beta2.ProcessClassLog) + }).Should(BeNumerically("==", 10)) + }) + }) + + FWhen("setting 2 logs per disk to use Transaction process", func() { + var initialLogServerPerPod, expectedPodCnt, expectedLogProcessesCnt int + + BeforeEach(func() { + initialLogServerPerPod = fdbCluster.GetLogServersPerPod() + initialPods := fdbCluster.GetLogPods() + expectedPodCnt = len(initialPods.Items) + expectedLogProcessesCnt = expectedPodCnt * initialLogServerPerPod + log.Printf( + "expectedPodCnt: %d, expectedProcessesCnt: %d", + expectedPodCnt, + expectedLogProcessesCnt, + ) + Eventually(func() int { + return len(fdbCluster.GetLogPods().Items) + }).Should(BeNumerically("==", expectedPodCnt)) + }) + + AfterEach(func() { + log.Printf("set log servers per Pod to %d", initialLogServerPerPod) + Expect(fdbCluster.SetLogServersPerPod(initialLogServerPerPod)).ShouldNot(HaveOccurred()) + log.Printf( + "expectedPodCnt: %d, expectedProcessesCnt: %d", expectedPodCnt, expectedPodCnt*initialLogServerPerPod, ) + Eventually(func() int { + return len(fdbCluster.GetLogPods().Items) + }).Should(BeNumerically("==", expectedPodCnt)) }) It("should update the log servers to the expected amount", func() { serverPerPod := initialLogServerPerPod * 2 - log.Printf("set storage server per Pod to %d", serverPerPod) + Expect(fdbCluster.SetTransactionServerPerPod(serverPerPod, 5)).ShouldNot(HaveOccurred()) Expect(fdbCluster.SetLogServersPerPod(serverPerPod)).ShouldNot(HaveOccurred()) log.Printf( "expectedPodCnt: %d, expectedStorageProcessesCnt: %d", expectedPodCnt, expectedPodCnt*serverPerPod, ) - validateProcessesCount(fdbCluster, fdbv1beta2.ProcessRoleLog, expectedPodCnt, expectedPodCnt*serverPerPod) + + Eventually(func() int { + return len(fdbCluster.GetTransactionPods().Items) + }).Should(BeNumerically("==", expectedPodCnt)) + Eventually(func() int { + return fdbCluster.GetProcessCountByProcessClass(fdbv1beta2.ProcessClassTransaction) + }).Should(BeNumerically("==", 10)) }) }) diff --git a/e2e/test_operator_upgrades/operator_upgrades_test.go b/e2e/test_operator_upgrades/operator_upgrades_test.go index 8e8f80205..d9c8cf686 100644 --- a/e2e/test_operator_upgrades/operator_upgrades_test.go +++ b/e2e/test_operator_upgrades/operator_upgrades_test.go @@ -1135,4 +1135,65 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() { EntryDescription("Upgrade from %[1]s to %[2]s"), fixtures.GenerateUpgradeTableEntries(testOptions), ) + + DescribeTable( + "with 2 log servers per Pod", + func(beforeVersion string, targetVersion string) { + clusterSetupWithConfig(beforeVersion, true, &fixtures.ClusterConfig{ + DebugSymbols: false, + LogServersPerPod: 2, + }) + + Expect(fdbCluster.UpgradeCluster(targetVersion, false)).NotTo(HaveOccurred()) + // Make sure the cluster is still running with 2 storage server per Pod. + Expect(fdbCluster.GetCluster().Spec.LogServersPerPod).To(Equal(2)) + + if !fixtures.VersionsAreProtocolCompatible(beforeVersion, targetVersion) { + // Ensure that the operator is setting the IncorrectConfigMap and IncorrectCommandLine conditions during the upgrade + // process. + expectedConditions := map[fdbv1beta2.ProcessGroupConditionType]bool{ + fdbv1beta2.IncorrectConfigMap: true, + fdbv1beta2.IncorrectCommandLine: true, + } + Eventually(func() bool { + cluster := fdbCluster.GetCluster() + + for _, processGroup := range cluster.Status.ProcessGroups { + if !processGroup.MatchesConditions(expectedConditions) { + return false + } + } + + return true + }).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).Should(BeTrue()) + } + + transactionSystemProcessGroups := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None) + // Wait until the cluster is upgraded and fully reconciled. + Expect(fdbCluster.WaitUntilWithForceReconcile(2, 600, func(cluster *fdbv1beta2.FoundationDBCluster) bool { + for _, processGroup := range cluster.Status.ProcessGroups { + if processGroup.ProcessClass == fdbv1beta2.ProcessClassLog { + continue + } + + transactionSystemProcessGroups[processGroup.ProcessGroupID] = fdbv1beta2.None{} + } + + // Allow soft reconciliation and make sure the running version was updated + return cluster.Status.Generations.Reconciled == cluster.Generation && cluster.Status.RunningVersion == targetVersion + })).NotTo(HaveOccurred()) + + // Get the desired process counts based on the current cluster configuration + processCounts, err := fdbCluster.GetProcessCounts() + Expect(err).NotTo(HaveOccurred()) + + // During an upgrade we expect that the transaction system processes are replaced, so we expect to have seen + // 2 times the process counts for transaction system processes. Add a small buffer of 5 to allow automatic + // replacements during an upgrade. + expectedProcessCounts := (processCounts.Total()-processCounts.Log)*2 + 5 + Expect(len(transactionSystemProcessGroups)).To(BeNumerically("<=", expectedProcessCounts)) + }, + EntryDescription("Upgrade from %[1]s to %[2]s"), + fixtures.GenerateUpgradeTableEntries(testOptions), + ) })