Skip to content

Commit

Permalink
compute block length accounting for compression
Browse files Browse the repository at this point in the history
Taken from PR #1698 and commit da5f789
  • Loading branch information
mangalaman93 committed Feb 18, 2023
1 parent c569be1 commit 1375846
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 2 deletions.
3 changes: 3 additions & 0 deletions db.go
Original file line number Diff line number Diff line change
Expand Up @@ -755,6 +755,9 @@ var requestPool = sync.Pool{
}

func (db *DB) writeToLSM(b *request) error {
db.lock.RLock()
defer db.lock.RUnlock()

// We should check the length of b.Prts and b.Entries only when badger is not
// running in InMemory mode. In InMemory mode, we don't write anything to the
// value log and that's why the length of b.Ptrs will always be zero.
Expand Down
1 change: 0 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
Expand Down
12 changes: 11 additions & 1 deletion table/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,16 @@ func NewTableBuilder(opts Options) *Builder {
return b
}

func maxEncodedLen(ctype options.CompressionType, sz int) int {
switch ctype {
case options.Snappy:
return snappy.MaxEncodedLen(sz)
case options.ZSTD:
return y.ZSTDCompressBound(sz)
}
return sz
}

func (b *Builder) handleBlock() {
defer b.wg.Done()

Expand All @@ -178,7 +188,7 @@ func (b *Builder) handleBlock() {
// BlockBuf should always less than or equal to allocated space. If the blockBuf is greater
// than allocated space that means the data from this block cannot be stored in its
// existing location.
allocatedSpace := (item.end) + padding + 1
allocatedSpace := maxEncodedLen(b.opts.Compression, (item.end)) + padding + 1
y.AssertTrue(len(blockBuf) <= allocatedSpace)

// blockBuf was allocated on allocator. So, we don't need to copy it over.
Expand Down

0 comments on commit 1375846

Please sign in to comment.