Skip to content

Add MaxValueSize limit to verifiers #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mdogan merged 1 commit into from
Aug 11, 2026
Merged

Add MaxValueSize limit to verifiers #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func main() {
### Verifying a file

The following code demonstrates how to verify an RDB file is not corrupt, and
does not exceed the defined limits of the total data, max entry, and max key sizes.
does not exceed the defined limits of the total data, max entry, max value, and max key sizes.

```go
import (
Expand All @@ -98,6 +98,7 @@ func main() {
opts := rdb.VerifyFileOptions{
MaxDataSize: 256 << 20, // 256 MB
MaxEntrySize: 100 << 20, // 100 MB
MaxValueSize: 100 << 20, // 100 MB
MaxStreamPELSize: 1000,
}
err := rdb.VerifyFile("/path/to/dump.rdb", opts)
Expand All @@ -110,7 +111,7 @@ func main() {
### Verifying a reader

The following code demonstrates how to verify an io.Reader to reads an RDB file is not corrupt, and
does not exceed the defined limits of the total data, max entry, and max key sizes.
does not exceed the defined limits of the total data, max entry, max value, and max key sizes.

```go
import (
Expand All @@ -124,6 +125,7 @@ func main() {
opts := rdb.VerifyReaderOptions{
MaxDataSize: 256 << 20, // 256 MB
MaxEntrySize: 100 << 20, // 100 MB
MaxValueSize: 100 << 20, // 100 MB
MaxStreamPELSize: 1000,
}

Expand All @@ -140,7 +142,7 @@ func main() {
### Verifying a value

The following code demonstrates how to verify an RDB value is not corrupt, and
does not exceed the defined limits of the max entry size.
does not exceed the defined limits of the max entry and max value sizes.

```go
import (
Expand All @@ -152,6 +154,7 @@ import (
func main() {
opts := rdb.VerifyValueOptions{
MaxEntrySize: 100 << 20, // 100 MB
MaxValueSize: 100 << 20, // 100 MB
MaxStreamPELSize: 1000,
}
payload := []byte{ /*RDB value payload*/ }
Expand Down
8 changes: 6 additions & 2 deletions stream_reader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,9 @@ func (r *valueReader) readStreamEntries(cb func(StreamEntry) error) error {
ID: id,
Value: fields,
}
cb(entry)
if err := cb(entry); err != nil {
return err
}
}
} else {
numFieldsS, err := lpReader.readListpackEntry()
Expand Down Expand Up @@ -393,7 +395,9 @@ func (r *valueReader) readStreamEntries(cb func(StreamEntry) error) error {
ID: id,
Value: fields,
}
cb(entry)
if err := cb(entry); err != nil {
return err
}
}
}

Expand Down
76 changes: 72 additions & 4 deletions verify.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

var defaultMaxDataSize = 256 << 20 // 256 MB
var defaultMaxEntrySize = 100 << 20 // 100 MB
var defaultMaxValueSize = 100 << 20 // 100 MB
var defaultMaxKeySize = 32 << 10 // 32 KB
var defaultMaxStreamPELSize = 1000
var defaultMaxLibrarySize = 100 << 20 // 100 MB
Expand All @@ -19,6 +20,7 @@ const maxStreamStrSize = math.MaxUint32
type VerifyFileOptions struct {
MaxDataSize int
MaxEntrySize int
MaxValueSize int
MaxKeySize int
MaxStreamPELSize int
MaxLibrarySize int
Expand All @@ -35,6 +37,10 @@ func (o *VerifyFileOptions) maybeSetDefaults() {
o.MaxEntrySize = defaultMaxEntrySize
}

if o.MaxValueSize <= 0 {
o.MaxValueSize = defaultMaxValueSize
}

if o.MaxKeySize <= 0 {
o.MaxKeySize = defaultMaxKeySize
}
Expand All @@ -55,6 +61,7 @@ func VerifyFile(path string, opts VerifyFileOptions) error {
v := &verifier{
maxDataSize: opts.MaxDataSize,
maxEntrySize: opts.MaxEntrySize,
maxValueSize: opts.MaxValueSize,
maxKeySize: opts.MaxKeySize,
maxStreamPELSize: opts.MaxStreamPELSize,
maxLibrarySize: opts.MaxLibrarySize,
Expand Down Expand Up @@ -82,6 +89,7 @@ func VerifyFile(path string, opts VerifyFileOptions) error {
type VerifyReaderOptions struct {
MaxDataSize int
MaxEntrySize int
MaxValueSize int
MaxKeySize int
MaxStreamPELSize int
MaxLibrarySize int
Expand All @@ -98,6 +106,10 @@ func (o *VerifyReaderOptions) maybeSetDefaults() {
o.MaxEntrySize = defaultMaxEntrySize
}

if o.MaxValueSize <= 0 {
o.MaxValueSize = defaultMaxValueSize
}

if o.MaxKeySize <= 0 {
o.MaxKeySize = defaultMaxKeySize
}
Expand All @@ -118,6 +130,7 @@ func VerifyReader(r io.Reader, opts VerifyReaderOptions) error {
v := &verifier{
maxDataSize: opts.MaxDataSize,
maxEntrySize: opts.MaxEntrySize,
maxValueSize: opts.MaxValueSize,
maxKeySize: opts.MaxKeySize,
maxStreamPELSize: opts.MaxStreamPELSize,
maxLibrarySize: opts.MaxLibrarySize,
Expand All @@ -132,6 +145,7 @@ func VerifyReader(r io.Reader, opts VerifyReaderOptions) error {

type VerifyValueOptions struct {
MaxEntrySize int
MaxValueSize int
MaxStreamPELSize int
}

Expand All @@ -140,6 +154,10 @@ func (o *VerifyValueOptions) maybeSetDefaults() {
o.MaxEntrySize = defaultMaxEntrySize
}

if o.MaxValueSize <= 0 {
o.MaxValueSize = defaultMaxValueSize
}

if o.MaxStreamPELSize <= 0 {
o.MaxStreamPELSize = defaultMaxStreamPELSize
}
Expand All @@ -151,6 +169,7 @@ func VerifyValue(payload []byte, opts VerifyValueOptions) error {
opts.maybeSetDefaults()
v := &verifier{
maxEntrySize: opts.MaxEntrySize,
maxValueSize: opts.MaxValueSize,
maxStreamPELSize: opts.MaxStreamPELSize,
// We don't care about the values below, as they don't
// really apply to RDB values.
Expand All @@ -170,6 +189,10 @@ func errMaxEntrySizeExceeded(current int, limit int) error {
return fmt.Errorf("max entry size is exceeded. current: %d, limit: %d", current, limit)
}

func errMaxValueSizeExceeded(current int, limit int) error {
return fmt.Errorf("max value size is exceeded. current: %d, limit: %d", current, limit)
}

func errMaxKeySizeExceeded(current int, limit int) error {
return fmt.Errorf("max key size is exceeded. current: %d, limit: %d", current, limit)
}
Expand All @@ -189,6 +212,7 @@ func errMaxLibrarySizeExceeded(current int, limit int) error {
type verifier struct {
maxDataSize int
maxEntrySize int
maxValueSize int
maxKeySize int
maxStreamPELSize int
allowPartialVerify bool
Expand All @@ -203,6 +227,10 @@ func (v *verifier) HandleString(key string, value string) error {
return errMaxKeySizeExceeded(len(key), v.maxKeySize)
}

if len(value) > v.maxValueSize {
return errMaxValueSizeExceeded(len(value), v.maxValueSize)
}

if len(value) > v.maxEntrySize {
return errMaxEntrySizeExceeded(len(value), v.maxEntrySize)
}
Expand Down Expand Up @@ -232,6 +260,10 @@ func (v *verifier) HashEntryHandler(key string) func(field string, value string)
var entrySize int
return func(field, value string) error {
elementSize := len(field) + len(value)
if elementSize > v.maxValueSize {
return errMaxValueSizeExceeded(elementSize, v.maxValueSize)
}

entrySize += elementSize
if entrySize > v.maxEntrySize {
return errMaxEntrySizeExceeded(entrySize, v.maxEntrySize)
Expand Down Expand Up @@ -263,6 +295,10 @@ func (v *verifier) HashWithExpEntryHandler(key string) func(field string, value
var entrySize int
return func(field, value string, exp time.Time) error {
elementSize := len(field) + len(value) + 8
if elementSize > v.maxValueSize {
return errMaxValueSizeExceeded(elementSize, v.maxValueSize)
}

entrySize += elementSize
if entrySize > v.maxEntrySize {
return errMaxEntrySizeExceeded(entrySize, v.maxEntrySize)
Expand Down Expand Up @@ -294,6 +330,10 @@ func (v *verifier) ListEntryHandler(key string) func(elem string) error {
var entrySize int
return func(elem string) error {
elementSize := len(elem)
if elementSize > v.maxValueSize {
return errMaxValueSizeExceeded(elementSize, v.maxValueSize)
}

entrySize += elementSize
if entrySize > v.maxEntrySize {
return errMaxEntrySizeExceeded(entrySize, v.maxEntrySize)
Expand Down Expand Up @@ -325,6 +365,10 @@ func (v *verifier) SetEntryHandler(key string) func(elem string) error {
var entrySize int
return func(elem string) error {
elementSize := len(elem)
if elementSize > v.maxValueSize {
return errMaxValueSizeExceeded(elementSize, v.maxValueSize)
}

entrySize += elementSize
if entrySize > v.maxEntrySize {
return errMaxEntrySizeExceeded(entrySize, v.maxEntrySize)
Expand Down Expand Up @@ -356,6 +400,10 @@ func (v *verifier) ZsetEntryHandler(key string) func(elem string, score float64)
var entrySize int
return func(elem string, score float64) error {
elementSize := len(elem) + 8
if elementSize > v.maxValueSize {
return errMaxValueSizeExceeded(elementSize, v.maxValueSize)
}

entrySize += elementSize
if entrySize > v.maxEntrySize {
return errMaxEntrySizeExceeded(entrySize, v.maxEntrySize)
Expand Down Expand Up @@ -387,6 +435,10 @@ func (v *verifier) ArrayEntryHandler(key string) func(index uint64, value string
var entrySize int
return func(index uint64, value string) error {
elementSize := len(value) + 8 // 8: index
if elementSize > v.maxValueSize {
return errMaxValueSizeExceeded(elementSize, v.maxValueSize)
}

entrySize += elementSize
if entrySize > v.maxEntrySize {
return errMaxEntrySizeExceeded(entrySize, v.maxEntrySize)
Expand All @@ -406,6 +458,10 @@ func (v *verifier) HandleModule(key string, value string, marker ModuleMarker) e
return errMaxKeySizeExceeded(len(key), v.maxKeySize)
}

if len(value) > v.maxValueSize {
return errMaxValueSizeExceeded(len(value), v.maxValueSize)
}

if len(value) > v.maxEntrySize {
return errMaxEntrySizeExceeded(len(value), v.maxEntrySize)
}
Expand Down Expand Up @@ -442,11 +498,17 @@ func (v *verifier) StreamEntryHandler(key string) func(entry StreamEntry) error
valueSize += len(value)
}

entrySize := valueSize + 16 // 8: ID#Seq + 8: ID#Millis
if entrySize > v.maxValueSize {
return errMaxValueSizeExceeded(entrySize, v.maxValueSize)
}

// we don't check for the max entry size here as we store
// stream entries on disk.
// the max value size is still checked above,
// as it applies to a single stream entry.

v.dataSize += valueSize
v.dataSize += 16 // 8: ID#Seq + 8: ID#Millis
v.dataSize += entrySize
if v.dataSize > v.maxDataSize {
return errMaxDataSizeExceeded(v.dataSize, v.maxDataSize)
}
Expand Down Expand Up @@ -477,15 +539,21 @@ func (v *verifier) StreamGroupHandler(key string) func(group StreamConsumerGroup
}

for _, pe := range consumer.PendingEntries {
groupSize += 32 // 8: ID#Seq + 8: ID#Millis + 8: DeliveryCount + 8: DeliveryTime
pendingEntrySize := 32 // 8: ID#Seq + 8: ID#Millis + 8: DeliveryCount + 8: DeliveryTime

for _, val := range pe.Entry.Value {
if len(val) > maxStreamStrSize {
return errMaxStreamStrSizeExceeded(len(val), maxStreamStrSize)
}

groupSize += len(val)
pendingEntrySize += len(val)
}

if pendingEntrySize > v.maxValueSize {
return errMaxValueSizeExceeded(pendingEntrySize, v.maxValueSize)
}

groupSize += pendingEntrySize
}
}
entrySize += groupSize
Expand Down
Loading
Loading