fix correct backup store initiation (#2465) (#2470)

Co-authored-by: Gerard Snaauw <[email protected]>

docs/pages/release_notes.rst

@@ -12,6 +12,7 @@ Release date: 2023-09-07
 - Fixed an issue where revocations received through the network were not written to a backup that was introduced in v5.4.0.
 Nodes upgrading from v5.4.0-v5.4.2 need to make an empty POST call to ``<node-address>/internal/network/v1/reprocess?type=application/ld+json%3Btype=revocation``.
 - Reduced number of pages transmitted per message on a full sync to enhance performance
+- Fixed a performance issue with initializing the backup databases
 - Fixed some typos in NL language templates (@jelmerterwal)
 
 **Full Changelog**: https://github.com/nuts-foundation/nuts-node/compare/v5.4.2...v5.4.3

storage/leia.go

@@ -127,7 +127,7 @@ func (k *kvBackedLeiaStore) handleRestore(config LeiaBackupConfiguration) error
 		ref leia.Reference
 		doc leia.Document
 	}
-	var set []refDoc
+
 	writeDocuments := func(set []refDoc) error {
 		return k.backup.Write(context.Background(), func(tx stoabs.WriteTx) error {
 			writer := tx.GetShelfWriter(config.BackupShelf)
@@ -136,15 +136,17 @@ func (k *kvBackedLeiaStore) handleRestore(config LeiaBackupConfiguration) error
 					return err
 				}
 			}
-			set = make([]refDoc, 0)
 			return nil
 		})
 	}
 
+	set := make([]refDoc, 0, limit)
 	err := collection.Iterate(query, func(ref leia.Reference, value []byte) error {
 		set = append(set, refDoc{ref: ref, doc: value})
 		if len(set) >= limit {
-			return writeDocuments(set)
+			err := writeDocuments(set)
+			set = make([]refDoc, 0, limit)
+			return err
 		}
 		return nil
 	})

storage/leia_test.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"crypto/sha1"
 	"encoding/json"
+	"fmt"
 	"github.com/nuts-foundation/go-did/vc"
 	"github.com/nuts-foundation/go-leia/v4"
 	"github.com/nuts-foundation/go-stoabs"
@@ -30,6 +31,7 @@ import (
 	"github.com/nuts-foundation/nuts-node/test/io"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/mock/gomock"
 	"path"
 	"testing"
 )
@@ -120,6 +122,42 @@ func Test_leiaIssuerStore_handleRestore(t *testing.T) {
 			})
 		})
 	}
+
+	t.Run("exact number of documents is written", func(t *testing.T) {
+		backupConfig := LeiaBackupConfiguration{
+			BackupShelf:    "JSON",
+			CollectionName: "JSON",
+			CollectionType: leia.JSONCollection,
+			SearchQuery:    leia.NewJSONPath("id"),
+		}
+		documentSet := make([]leia.Document, 102)
+		for i := 0; i < 102; i++ {
+			documentSet[i] = []byte(fmt.Sprintf("{\"id\":\"%d\"}", i))
+		}
+		testDir := io.TestDirectory(t)
+		issuerStorePath := path.Join(testDir, "vcr", "private-credentials.db")
+		leiaStore, err := leia.NewStore(issuerStorePath)
+		require.NoError(t, err)
+		ctrl := gomock.NewController(t)
+		mockBackup := stoabs.NewMockKVStore(ctrl)
+		store := kvBackedLeiaStore{
+			store:               leiaStore,
+			backup:              mockBackup,
+			collectionConfigSet: nil,
+		}
+		collection := store.store.Collection(leia.JSONCollection, backupConfig.CollectionName)
+		idIndex := collection.NewIndex("byID", leia.NewFieldIndexer(backupConfig.SearchQuery))
+		err = collection.AddIndex(idIndex)
+		require.NoError(t, err)
+		err = collection.Add(documentSet)
+		require.NoError(t, err)
+		mockBackup.EXPECT().ReadShelf(ctx, backupConfig.BackupShelf, gomock.Any())
+		mockBackup.EXPECT().Write(context.Background(), gomock.Any()).Times(2)
+
+		err = store.handleRestore(backupConfig)
+
+		require.NoError(t, err)
+	})
 }
 
 func assertCredential(t *testing.T, store *kvBackedLeiaStore, config LeiaBackupConfiguration, expected vc.VerifiableCredential) {