From b8e9c65bc4d5ad47f9d61a08cd3ad326d61703a4 Mon Sep 17 00:00:00 2001 From: "Johannes M. Scheuermann" Date: Wed, 25 Sep 2024 17:39:38 +0200 Subject: [PATCH] Add basic backup and restore test with seaweedfs --- Dockerfile | 2 +- e2e/Makefile | 2 + e2e/fixtures/blobstore.go | 258 ++++++++++++++++++ e2e/fixtures/fdb_backup.go | 243 +++++++++++++++++ e2e/fixtures/fdb_cluster.go | 119 ++++++++ e2e/fixtures/fdb_operator_client.go | 2 - e2e/fixtures/fdb_restore.go | 72 +++++ e2e/fixtures/kubernetes_fixtures.go | 21 ++ e2e/fixtures/options.go | 7 + e2e/fixtures/status.go | 78 ++++++ .../operator_backup_test.go | 86 ++++++ e2e/test_operator_backups/suite_test.go | 35 +++ 12 files changed, 922 insertions(+), 3 deletions(-) create mode 100644 e2e/fixtures/blobstore.go create mode 100644 e2e/fixtures/fdb_backup.go create mode 100644 e2e/fixtures/fdb_restore.go create mode 100644 e2e/test_operator_backups/operator_backup_test.go create mode 100644 e2e/test_operator_backups/suite_test.go diff --git a/Dockerfile b/Dockerfile index fef2ce61..2d25b688 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ WORKDIR / RUN set -eux && \ curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm && \ curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \ - microdnf install -y glibc && \ + microdnf install -y glibc pkg-config && \ microdnf clean all && \ # TODO(johscheuer): The 6.2.29 sha256 file is not well formatted, enable this check again once 7.1 is used as base. \ # sha256sum -c foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \ diff --git a/e2e/Makefile b/e2e/Makefile index 7da25916..a7b3b647 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -26,6 +26,7 @@ CHAOS_NAMESPACE?=chaos-testing STORAGE_CLASS?= STORAGE_ENGINE?= DUMP_OPERATOR_STATE?=true +SEAWEEDFS_IMAGE?=chrislusf/seaweedfs:3.73 # Defines the cloud provider used for the underlying Kubernetes cluster. Currently only kind is support, other cloud providers # should still work but this test framework has no special cases for those. CLOUD_PROVIDER?= @@ -158,4 +159,5 @@ nightly-tests: run --fdb-version-tag-mapping=$(FDB_VERSION_TAG_MAPPING) \ --unified-fdb-image=$(UNIFIED_FDB_IMAGE) \ --feature-server-side-apply=$(FEATURE_SERVER_SIDE_APPLY) \ + --seaweedfs-image=$(SEAWEEDFS_IMAGE) \ | grep -v 'constructing many client instances from the same exec auth config can cause performance problems during cert rotation' &> $(BASE_DIR)/../logs/$<.log diff --git a/e2e/fixtures/blobstore.go b/e2e/fixtures/blobstore.go new file mode 100644 index 00000000..55931b29 --- /dev/null +++ b/e2e/fixtures/blobstore.go @@ -0,0 +1,258 @@ +/* + * blobstore.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2018-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fixtures + +import ( + "bytes" + "context" + "errors" + "io" + "log" + "text/template" + "time" + + "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + seaweedFSName = "seaweedfs" + seaweedFSDeployment = `apiVersion: v1 +kind: Service +metadata: + name: seaweedfs + namespace: {{ .Namespace }} +spec: + type: ClusterIP + ports: + - port: 8333 + targetPort: 8333 + protocol: TCP + selector: + app: seaweedfs +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: seaweedfs + namespace: {{ .Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: seaweedfs-s3-secret + namespace: {{ .Namespace }} +stringData: + admin_access_key_id: "seaweedfs" + admin_secret_access_key: "totallys3cure" + seaweedfs_s3_config: | + { + "identities": [ + { + "name": "anvAdmin", + "credentials": [ + { + "accessKey": "seaweedfs", + "secretKey": "tot4llys3cure" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + } + ] + } + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: seaweedfs + name: seaweedfs + namespace: {{ .Namespace }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: seaweedfs + template: + metadata: + labels: + app: seaweedfs + spec: + containers: + - image: {{ .Image }} + imagePullPolicy: Always + args: + - server + - -dir=/data + - -s3 + - -s3.config=/etc/sw/seaweedfs_s3_config + name: manager + ports: + - containerPort: 8333 + name: seaweedfs + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "1" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /tmp + name: tmp + - mountPath: /etc/sw + name: users-config + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: data + persistentVolumeClaim: + claimName: seaweedfs + - name: tmp + emptyDir: {} + - name: users-config + secret: + defaultMode: 420 + secretName: seaweedfs-s3-secret +` +) + +// blobstoreConfig represents the configuration of the blobstore deployment. +type blobstoreConfig struct { + // Image represents the seaweedfs image that should be used in the Deployment. + Image string + // Namespace represents the namespace for the deployment and all associated resources + Namespace string +} + +// CreateBlobstoreIfAbsent creates the blobstore Deployment based on the template. +func (factory *Factory) CreateBlobstoreIfAbsent(namespace string) { + seaweedFSDeploymentTemplate, err := template.New("seaweedFSDeployment").Parse(seaweedFSDeployment) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + buf := bytes.Buffer{} + gomega.Expect(seaweedFSDeploymentTemplate.Execute(&buf, &blobstoreConfig{ + Image: prependRegistry(factory.options.registry, factory.options.seaweedFSImage), + Namespace: namespace, + })).NotTo(gomega.HaveOccurred()) + decoder := yamlutil.NewYAMLOrJSONDecoder(&buf, 100000) + + for { + var rawObj runtime.RawExtension + err := decoder.Decode(&rawObj) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + obj, _, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme). + Decode(rawObj.Raw, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + unstructuredObj := &unstructured.Unstructured{Object: unstructuredMap} + + gomega.Expect( + factory.CreateIfAbsent(unstructuredObj), + ).NotTo(gomega.HaveOccurred()) + } + + // Make sure the blobstore Pods are running before moving forward. + factory.waitUntilBlobstorePodsRunning(namespace) +} + +// waitUntilBlobstorePodsRunning waits until the blobstore Pods are running. +func (factory *Factory) waitUntilBlobstorePodsRunning(namespace string) { + deployment := &appsv1.Deployment{} + gomega.Expect( + factory.GetControllerRuntimeClient(). + Get(context.Background(), client.ObjectKey{Name: seaweedFSName, Namespace: namespace}, deployment), + ).NotTo(gomega.HaveOccurred()) + + expectedReplicas := int(pointer.Int32Deref(deployment.Spec.Replicas, 1)) + gomega.Eventually(func(g gomega.Gomega) int { + pods := factory.getBlobstorePods(namespace) + + log.Println("waiting for Pods to be running:", len(pods.Items), "expected:", expectedReplicas) + var runningReplicas int + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.DeletionTimestamp.IsZero() { + runningReplicas++ + continue + } + + // If the Pod is not running after 120 seconds we delete it and let the Deployment controller create a new Pod. + if time.Since(pod.CreationTimestamp.Time).Seconds() > 120.0 { + log.Println("seaweedfs Pod", pod.Name, "not running after 120 seconds, going to delete this Pod, status:", pod.Status) + err := factory.GetControllerRuntimeClient().Delete(context.Background(), &pod) + if k8serrors.IsNotFound(err) { + continue + } + + g.Expect(err).NotTo(gomega.HaveOccurred()) + } + } + + return runningReplicas + }).WithTimeout(10 * time.Minute).WithPolling(2 * time.Second).Should(gomega.BeNumerically(">=", expectedReplicas)) +} + +// getBlobstorePods returns the blobstore Pods in the provided namespace. +func (factory *Factory) getBlobstorePods(namespace string) *corev1.PodList { + pods := &corev1.PodList{} + gomega.Eventually(func() error { + return factory.GetControllerRuntimeClient(). + List(context.Background(), pods, client.InNamespace(namespace), client.MatchingLabels(map[string]string{"app": seaweedFSName})) + }).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).ShouldNot(gomega.HaveOccurred()) + + return pods +} diff --git a/e2e/fixtures/fdb_backup.go b/e2e/fixtures/fdb_backup.go new file mode 100644 index 00000000..54bfc01e --- /dev/null +++ b/e2e/fixtures/fdb_backup.go @@ -0,0 +1,243 @@ +/* + * fdb_backup.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2018-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fixtures + +import ( + "context" + "encoding/json" + "time" + + fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// FdbBackup represents a fdbv1beta2.FoundationDBBackup resource for doing backups of a FdbCluster. +type FdbBackup struct { + backup *fdbv1beta2.FoundationDBBackup + fdbCluster *FdbCluster +} + +// CreateBackupForCluster will create a FoundationDBBackup for the provided cluster. +func (factory *Factory) CreateBackupForCluster( + fdbCluster *FdbCluster, +) *FdbBackup { + // For more information how the backup system with the operator is working please look at + // the operator documentation: https://github.com/FoundationDB/fdb-kubernetes-operator/blob/master/docs/manual/backup.md + fdbVersion := factory.GetFDBVersion() + + backup := &fdbv1beta2.FoundationDBBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: fdbCluster.Name(), + Namespace: fdbCluster.Namespace(), + }, + Spec: fdbv1beta2.FoundationDBBackupSpec{ + AllowTagOverride: pointer.BoolPtr(true), + ClusterName: fdbCluster.Name(), + Version: fdbVersion.String(), + BlobStoreConfiguration: &fdbv1beta2.BlobStoreConfiguration{ + AccountName: "seaweedfs@seaweedfs:8333", + URLParameters: []fdbv1beta2.URLParameter{ + "secure_connection=0", + // The region must be specified since the blobstore URL doesn't have that information. + "region=us-east-1", + }, + }, + CustomParameters: fdbv1beta2.FoundationDBCustomParameters{ + // Enable if you want to get http debug logs. + // "knob_http_verbose_level=10", + }, + ImageType: fdbCluster.cluster.Spec.ImageType, + MainContainer: fdbCluster.cluster.Spec.MainContainer, + SidecarContainer: fdbCluster.cluster.Spec.SidecarContainer, + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: fdbv1beta2.MainContainerName, + Env: []corev1.EnvVar{ + { + Name: "FDB_TLS_CERTIFICATE_FILE", + Value: "/tmp/fdb-certs/tls.crt", + }, + { + Name: "FDB_TLS_CA_FILE", + Value: "/tmp/fdb-certs/ca.pem", + }, + { + Name: "FDB_TLS_KEY_FILE", + Value: "/tmp/fdb-certs/tls.key", + }, + { + Name: "FDB_BLOB_CREDENTIALS", + Value: "/tmp/backup-credentials/credentials", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "fdb-certs", + ReadOnly: true, + MountPath: "/tmp/fdb-certs", + }, + { + Name: "backup-credentials", + ReadOnly: true, + MountPath: "/tmp/backup-credentials", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "fdb-certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: factory.GetSecretName(), + }, + }, + }, + { + Name: "backup-credentials", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: factory.GetBackupSecretName(), + }, + }, + }, + }, + }, + }, + }, + } + + gomega.Expect(factory.CreateIfAbsent(backup)).NotTo(gomega.HaveOccurred()) + + factory.AddShutdownHook(func() error { + err := factory.GetControllerRuntimeClient().Delete(context.Background(), backup) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + + return nil + }) + + curBackup := &FdbBackup{ + backup: backup, + fdbCluster: fdbCluster, + } + curBackup.WaitForReconciliation() + return curBackup +} + +func (fdbBackup *FdbBackup) setState(state fdbv1beta2.BackupState) { + objectKey := client.ObjectKeyFromObject(fdbBackup.backup) + foundationDBBackup := &fdbv1beta2.FoundationDBBackup{} + gomega.Expect(fdbBackup.fdbCluster.factory.GetControllerRuntimeClient(). + Get(context.Background(), objectKey, foundationDBBackup)).NotTo(gomega.HaveOccurred()) + + // Backup is already in desired state + if foundationDBBackup.Spec.BackupState == state { + return + } + + foundationDBBackup.Spec.BackupState = state + gomega.Expect(fdbBackup.fdbCluster.factory.GetControllerRuntimeClient(). + Update(context.Background(), foundationDBBackup)).NotTo(gomega.HaveOccurred()) + fdbBackup.backup = foundationDBBackup + fdbBackup.WaitForReconciliation() +} + +// Stop will stop the FdbBackup. +func (fdbBackup *FdbBackup) Stop() { + fdbBackup.setState(fdbv1beta2.BackupStateStopped) +} + +// Start will start the FdbBackup. +func (fdbBackup *FdbBackup) Start() { + fdbBackup.setState(fdbv1beta2.BackupStateRunning) +} + +// Pause will pause the FdbBackup. +func (fdbBackup *FdbBackup) Pause() { + fdbBackup.setState(fdbv1beta2.BackupStatePaused) +} + +// WaitForReconciliation waits until the FdbBackup resource is fully reconciled. +func (fdbBackup *FdbBackup) WaitForReconciliation() { + objectKey := client.ObjectKeyFromObject(fdbBackup.backup) + + gomega.Eventually(func(g gomega.Gomega) bool { + curBackup := &fdbv1beta2.FoundationDBBackup{} + g.Expect(fdbBackup.fdbCluster.factory.GetControllerRuntimeClient(). + Get(context.Background(), objectKey, curBackup)).NotTo(gomega.HaveOccurred()) + + return curBackup.Status.Generations.Reconciled == curBackup.ObjectMeta.Generation + }).WithTimeout(15*time.Minute).WithPolling(2*time.Second).Should(gomega.BeTrue(), "error waiting for reconciliation") +} + +// WaitForRestorableVersion will wait until the back is restorable. +func (fdbBackup *FdbBackup) WaitForRestorableVersion(version uint64) { + gomega.Eventually(func(g gomega.Gomega) uint64 { + backupPod := fdbBackup.GetBackupPod() + out, _, err := fdbBackup.fdbCluster.ExecuteCmdOnPod( + *backupPod, + fdbv1beta2.MainContainerName, + "fdbbackup status --json", + false, + ) + g.Expect(err).NotTo(gomega.HaveOccurred()) + var result map[string]interface{} + g.Expect(json.Unmarshal([]byte(out), &result)).NotTo(gomega.HaveOccurred()) + + restorable, ok := result["Restorable"].(bool) + g.Expect(ok).To(gomega.BeTrue()) + g.Expect(restorable).To(gomega.BeTrue()) + + restorablePoint, ok := result["LatestRestorablePoint"].(map[string]interface{}) + g.Expect(ok).To(gomega.BeTrue()) + + restorableVersion, ok := restorablePoint["Version"].(float64) + g.Expect(ok).To(gomega.BeTrue()) + + return uint64(restorableVersion) + }).WithTimeout(10*time.Minute).WithPolling(2*time.Second).Should(gomega.BeNumerically(">", version), "error waiting for restorable version") +} + +// GetBackupPod returns a random backup Pod for the provided backup. +func (fdbBackup *FdbBackup) GetBackupPod() *corev1.Pod { + return fdbBackup.fdbCluster.factory.ChooseRandomPod(fdbBackup.GetBackupPods()) +} + +// GetBackupPods returns a *corev1.PodList, which contains all pods for the provided backup. +func (fdbBackup *FdbBackup) GetBackupPods() *corev1.PodList { + podList := &corev1.PodList{} + + gomega.Expect(fdbBackup.fdbCluster.factory.GetControllerRuntimeClient().List(context.Background(), podList, + client.InNamespace(fdbBackup.fdbCluster.Namespace()), + client.MatchingLabels(map[string]string{fdbv1beta2.BackupDeploymentPodLabel: fdbBackup.fdbCluster.Name() + "-backup-agents"}), + )).NotTo(gomega.HaveOccurred()) + + return podList +} diff --git a/e2e/fixtures/fdb_cluster.go b/e2e/fixtures/fdb_cluster.go index 6077fdff..9668fd87 100644 --- a/e2e/fixtures/fdb_cluster.go +++ b/e2e/fixtures/fdb_cluster.go @@ -1727,3 +1727,122 @@ func (fdbCluster *FdbCluster) CreateTesterDeployment(replicas int) *appsv1.Deplo return deploy } + +// GetClusterVersion returns the cluster's version +func (fdbCluster *FdbCluster) GetClusterVersion() uint64 { + stdout, _, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry("getversion", false, 30) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + version, err := strconv.ParseUint(strings.TrimSpace(stdout), 10, 64) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + return version +} + +// ClearRange will delete the provided range. +func (fdbCluster *FdbCluster) ClearRange(prefixBytes []byte, timeout int) { + begin := FdbPrintable(prefixBytes) + endBytes, err := FdbStrinc(prefixBytes) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + end := FdbPrintable(endBytes) + _, stderr, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry(fmt.Sprintf( + "writemode on; clearrange %s %s", + begin, + end, + ), false, timeout) + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr) +} + +// KeyValue represents a key and value that can be stored in FDB. +type KeyValue struct { + Key []byte + Value []byte +} + +// GetKey returns all the printable characters of the key. +func (keyValue *KeyValue) GetKey() string { + return FdbPrintable(keyValue.Key) +} + +// GetValue returns all printable characters of the value. +func (keyValue *KeyValue) GetValue() string { + return FdbPrintable(keyValue.Value) +} + +// GetRange will return the values of the provided range. +func (fdbCluster *FdbCluster) GetRange( + prefixBytes []byte, + limit int, + timeout int, +) (keyValues []KeyValue) { + endBytes, err := FdbStrinc(prefixBytes) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + stdout, _, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry(fmt.Sprintf( + "option on ACCESS_SYSTEM_KEYS; getrange %s %s %d", + FdbPrintable(prefixBytes), + FdbPrintable(endBytes), + limit, + ), false, timeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + for _, line := range strings.Split(strings.TrimSuffix(stdout, "\n"), "\n") { + line = strings.TrimSpace(line) + sep := "' is `" + idx := strings.Index(line, sep) + if idx != -1 { + key, parseErr := Unprintable(line[1:idx]) // Remove the first "`" + gomega.Expect(parseErr).NotTo(gomega.HaveOccurred()) + value, parseErr := Unprintable(line[idx+len(sep) : len(line)-1]) // Remove the last "'" + gomega.Expect(parseErr).NotTo(gomega.HaveOccurred()) + keyValues = append(keyValues, KeyValue{ + Key: key, + Value: value, + }) + } + } + + return keyValues +} + +// GenerateRandomValues will generate n random values with the provided prefix. +func (fdbCluster *FdbCluster) GenerateRandomValues( + n int, + prefix byte, +) []KeyValue { + res := make([]KeyValue, 0, n) + index := []byte{'a'} + var err error + for i := 0; i < n; i++ { + res = append(res, KeyValue{ + Key: append([]byte{prefix}, index...), + Value: []byte(fdbCluster.factory.RandStringRunes(4)), + }) + index, err = FdbStrinc(index) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + return res +} + +// WriteKeyValue writes a single key value pair into FDB. +func (fdbCluster *FdbCluster) WriteKeyValue( + keyValue KeyValue, + timeout int, +) { + _, stderr, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry( + fmt.Sprintf("writemode on; set %s %s", keyValue.GetKey(), keyValue.GetValue()), + false, + timeout, + ) + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr) +} + +// WriteKeyValues writes multiples key values into FDB. +func (fdbCluster *FdbCluster) WriteKeyValues(keyValues []KeyValue) { + for _, kv := range keyValues { + fdbCluster.WriteKeyValue(kv, 30) + } +} diff --git a/e2e/fixtures/fdb_operator_client.go b/e2e/fixtures/fdb_operator_client.go index ea1de670..a3bbeddf 100644 --- a/e2e/fixtures/fdb_operator_client.go +++ b/e2e/fixtures/fdb_operator_client.go @@ -337,7 +337,6 @@ spec: - name: backup-credentials secret: secretName: {{ .BackupSecretName }} - optional: true - name: fdb-certs secret: secretName: {{ .SecretName }} @@ -472,7 +471,6 @@ spec: - name: backup-credentials secret: secretName: {{ .BackupSecretName }} - optional: true - name: fdb-certs secret: secretName: {{ .SecretName }} diff --git a/e2e/fixtures/fdb_restore.go b/e2e/fixtures/fdb_restore.go new file mode 100644 index 00000000..b9dfb112 --- /dev/null +++ b/e2e/fixtures/fdb_restore.go @@ -0,0 +1,72 @@ +/* + * fdb_restore.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2018-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fixtures + +import ( + "context" + "time" + + fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2" + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CreateRestoreForCluster will create a FoundationDBRestore resource based on the provided backup resource. +// For more information how the backup system with the operator is working please look at +// the operator documentation: https://github.com/FoundationDB/fdb-kubernetes-operator/blob/master/docs/manual/backup.md +func (factory *Factory) CreateRestoreForCluster(backup *FdbBackup) { + gomega.Expect(backup).NotTo(gomega.BeNil()) + restore := &fdbv1beta2.FoundationDBRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: backup.fdbCluster.Name(), + Namespace: backup.fdbCluster.Namespace(), + }, + Spec: fdbv1beta2.FoundationDBRestoreSpec{ + DestinationClusterName: backup.fdbCluster.Name(), + BlobStoreConfiguration: backup.backup.Spec.BlobStoreConfiguration, + CustomParameters: backup.backup.Spec.CustomParameters, + }, + } + gomega.Expect(factory.CreateIfAbsent(restore)).NotTo(gomega.HaveOccurred()) + + factory.AddShutdownHook(func() error { + return factory.GetControllerRuntimeClient().Delete(context.Background(), restore) + }) + + waitForRestoreToComplete(backup) +} + +// waitForRestoreToComplete waits until the restore completed. +func waitForRestoreToComplete(backup *FdbBackup) { + gomega.Eventually(func(g gomega.Gomega) string { + backupPod := backup.GetBackupPod() + + out, _, err := backup.fdbCluster.ExecuteCmdOnPod( + *backupPod, + fdbv1beta2.MainContainerName, + "fdbrestore status --dest_cluster_file $FDB_CLUSTER_FILE", + false, + ) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + return out + }).WithTimeout(20 * time.Minute).WithPolling(2 * time.Second).Should(gomega.ContainSubstring("State: completed")) +} diff --git a/e2e/fixtures/kubernetes_fixtures.go b/e2e/fixtures/kubernetes_fixtures.go index fded7166..16f05ef5 100644 --- a/e2e/fixtures/kubernetes_fixtures.go +++ b/e2e/fixtures/kubernetes_fixtures.go @@ -125,6 +125,27 @@ func (factory *Factory) createNamespace(suffix string) string { secret.SetResourceVersion("") gomega.Expect(factory.CreateIfAbsent(secret)).NotTo(gomega.HaveOccurred()) + // Create the backup credentials for backup related operations. + backupCredentials := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: factory.GetBackupSecretName(), + Namespace: namespace, + }, + StringData: map[string]string{ + "credentials": `{ + "accounts": { + "seaweedfs@seaweedfs": { + "secret" : "tot4llys3cure" + }, + "seaweedfs@seaweedfs:8333": { + "secret" : "tot4llys3cure" + } + } +}`, + }, + } + gomega.Expect(factory.CreateIfAbsent(backupCredentials)).NotTo(gomega.HaveOccurred()) + factory.ensureRBACSetupExists(namespace) gomega.Expect(factory.ensureFDBOperatorExists(namespace)).ToNot(gomega.HaveOccurred()) log.Printf("using namespace %s for testing", namespace) diff --git a/e2e/fixtures/options.go b/e2e/fixtures/options.go index db5cf440..aebaff24 100644 --- a/e2e/fixtures/options.go +++ b/e2e/fixtures/options.go @@ -38,6 +38,7 @@ type FactoryOptions struct { unifiedFDBImage string sidecarImage string operatorImage string + seaweedFSImage string dataLoaderImage string registry string fdbVersion string @@ -210,6 +211,12 @@ func (options *FactoryOptions) BindFlags(fs *flag.FlagSet) { "if defined, the test suite will use this information to map the image tag to the specified version. Multiple entries can be"+ "provided by separating them with a \",\". The mapping must have the format $version:$tag, e.g. 7.1.57:7.1.57-testing."+ "This option will only work for the main container with the split image (sidecar).") + fs.StringVar( + &options.seaweedFSImage, + "seaweedfs-image", + "chrislusf/seaweedfs:3.73", + "defines the seaweedfs image that should be used for testing. SeaweedFS is used for backup and restore testing to spin up a S3 compatible blobstore.", + ) } func (options *FactoryOptions) validateFlags() error { diff --git a/e2e/fixtures/status.go b/e2e/fixtures/status.go index 8f7b6c55..2f0dd233 100644 --- a/e2e/fixtures/status.go +++ b/e2e/fixtures/status.go @@ -460,5 +460,83 @@ func FdbPrintable(d []byte) string { } buf.WriteString(fmt.Sprintf("\\x%02x", b)) } + return buf.String() } + +// FdbStrinc returns the first key that would sort outside the range prefixed by +// // prefix, or an error if prefix is empty or contains only 0xFF bytes. +// Copied from foundationdb bindings/go/src/fdb/range.go func Strinc(prefix []byte) ([]byte, error) +func FdbStrinc(prefix []byte) ([]byte, error) { + for i := len(prefix) - 1; i >= 0; i-- { + if prefix[i] != 0xFF { + ret := make([]byte, i+1) + copy(ret, prefix[:i+1]) + ret[i]++ + return ret, nil + } + } + return nil, fmt.Errorf("key must contain at least one byte not equal to 0xFF") +} + +// Unprintable adapted from foundationdb fdbclient/NativeAPI.actor.cpp std::string unprintable(std::string const& val). +func Unprintable(val string) ([]byte, error) { + s := new(bytes.Buffer) + for i := 0; i < len(val); i++ { + c := val[i] + if c == '\\' { + i++ + if i == len(val) { + return nil, fmt.Errorf(fmt.Sprintf("end after one \\ when unprint [%s]", val)) + } + switch val[i] { + case '\\': + { + s.WriteByte('\\') + } + case 'x': + { + if i+2 >= len(val) { + return nil, fmt.Errorf( + fmt.Sprintf("not have two chars after \\x when unprint [%s]", val), + ) + } + d1, err := unhex(val[i+1]) + if err != nil { + return nil, err + } + d2, err := unhex(val[i+2]) + if err != nil { + return nil, err + } + s.WriteByte(byte((d1 << 4) + d2)) + i += 2 + } + default: + { + return nil, fmt.Errorf( + fmt.Sprintf("after \\ it's neither \\ nor x when unprint %s", val), + ) + } + } + } else { + s.WriteByte(c) + } + } + return s.Bytes(), nil +} + +// unhex adapted from foundationdb fdbclient/NativeAPI.actor.cpp std::string int unhex(char c). +func unhex(c byte) (int, error) { + if c >= '0' && c <= '9' { + return int(c - '0'), nil + } + if c >= 'a' && c <= 'f' { + return int(c - 'a' + 10), nil + } + if c >= 'A' && c <= 'F' { + return int(c - 'A' + 10), nil + } + + return -1, fmt.Errorf(fmt.Sprintf("failed to unhex %x", c)) +} diff --git a/e2e/test_operator_backups/operator_backup_test.go b/e2e/test_operator_backups/operator_backup_test.go new file mode 100644 index 00000000..36b2ad2d --- /dev/null +++ b/e2e/test_operator_backups/operator_backup_test.go @@ -0,0 +1,86 @@ +/* + * operator_backup_test.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2018-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operatorbackup + +/* +This test suite contains tests related to backup and restore with the operator. +*/ + +import ( + "log" + + "github.com/FoundationDB/fdb-kubernetes-operator/e2e/fixtures" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var ( + factory *fixtures.Factory + fdbCluster *fixtures.FdbCluster + testOptions *fixtures.FactoryOptions +) + +func init() { + testOptions = fixtures.InitFlags() +} + +var _ = BeforeSuite(func() { + factory = fixtures.CreateFactory(testOptions) + fdbCluster = factory.CreateFdbCluster( + fixtures.DefaultClusterConfig(false), + factory.GetClusterOptions()..., + ) + + // Create a blobstore for testing backups and restore + factory.CreateBlobstoreIfAbsent(fdbCluster.Namespace()) +}) + +var _ = AfterSuite(func() { + if CurrentSpecReport().Failed() { + log.Printf("failed due to %s", CurrentSpecReport().FailureMessage()) + } + factory.Shutdown() +}) + +var _ = Describe("Operator Backup", Label("e2e", "pr"), func() { + When("a cluster has backups enabled and then restored", func() { + var keyValues []fixtures.KeyValue + var prefix byte = 'a' + var backup *fixtures.FdbBackup + + BeforeEach(func() { + log.Println("creating backup for cluster") + backup = factory.CreateBackupForCluster(fdbCluster) + keyValues = fdbCluster.GenerateRandomValues(10, prefix) + fdbCluster.WriteKeyValues(keyValues) + clusterVersion := fdbCluster.GetClusterVersion() + backup.WaitForRestorableVersion(clusterVersion) + backup.Stop() + }) + + It("should restore the cluster successfully", func() { + fdbCluster.ClearRange([]byte{prefix}, 60) + factory.CreateRestoreForCluster(backup) + restoreValues := fdbCluster.GetRange([]byte{prefix}, 25, 60) + Expect(restoreValues).Should(Equal(keyValues)) + }) + }) +}) diff --git a/e2e/test_operator_backups/suite_test.go b/e2e/test_operator_backups/suite_test.go new file mode 100644 index 00000000..e93d909c --- /dev/null +++ b/e2e/test_operator_backups/suite_test.go @@ -0,0 +1,35 @@ +/* + * suite_test.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2018-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package operatorbackup + +import ( + "testing" + "time" + + "github.com/FoundationDB/fdb-kubernetes-operator/e2e/fixtures" + "github.com/onsi/gomega" +) + +func TestOperator(t *testing.T) { + gomega.SetDefaultEventuallyTimeout(10 * time.Second) + fixtures.SetTestSuiteName("operator-backup-test") + fixtures.RunGinkgoTests(t, "Operator Backup test suite") +}