Make optimised scan functionality optional (#68)

This commit is contained in:
Awn 2019-08-16 01:51:59 +01:00 committed by James Mills
parent 156d29e344
commit e8bee948bc
2 changed files with 54 additions and 10 deletions

@ -1,6 +1,7 @@
package bitcask
import (
"bytes"
"encoding/json"
"errors"
"hash/crc32"
@ -162,7 +163,10 @@ func (b *Bitcask) Put(key, value []byte) error {
}
item := b.keydir.Add(key, b.curr.FileID(), offset, n)
b.trie.Add(string(key), item)
if b.config.greedyScan {
b.trie.Add(string(key), item)
}
return nil
}
@ -176,7 +180,10 @@ func (b *Bitcask) Delete(key []byte) error {
}
b.keydir.Delete(key)
b.trie.Remove(string(key))
if b.config.greedyScan {
b.trie.Remove(string(key))
}
return nil
}
@ -185,12 +192,25 @@ func (b *Bitcask) Delete(key []byte) error {
// the function `f` with the keys found. If the function returns an error
// no further keys are processed and the first error returned.
func (b *Bitcask) Scan(prefix []byte, f func(key []byte) error) error {
keys := b.trie.PrefixSearch(string(prefix))
for _, key := range keys {
if err := f([]byte(key)); err != nil {
return err
if b.config.greedyScan {
keys := b.trie.PrefixSearch(string(prefix))
for _, key := range keys {
if err := f([]byte(key)); err != nil {
return err
}
}
return nil
}
keys := b.Keys()
for key := range keys {
if bytes.Equal(prefix, key[:len(prefix)]) {
if err := f([]byte(key)); err != nil {
return err
}
}
}
return nil
}
@ -295,7 +315,11 @@ func (b *Bitcask) reopen() error {
}
keydir := internal.NewKeydir()
trie := trie.New()
var t *trie.Trie
if b.config.greedyScan {
t = trie.New()
}
if internal.Exists(path.Join(b.path, "index")) {
if err := keydir.Load(path.Join(b.path, "index")); err != nil {
@ -303,7 +327,9 @@ func (b *Bitcask) reopen() error {
}
for key := range keydir.Keys() {
item, _ := keydir.Get(key)
trie.Add(string(key), item)
if b.config.greedyScan {
t.Add(string(key), item)
}
}
} else {
for i, df := range datafiles {
@ -323,7 +349,9 @@ func (b *Bitcask) reopen() error {
}
item := keydir.Add(e.Key, ids[i], e.Offset, n)
trie.Add(string(e.Key), item)
if b.config.greedyScan {
t.Add(string(e.Key), item)
}
}
}
}
@ -343,7 +371,9 @@ func (b *Bitcask) reopen() error {
b.keydir = keydir
b.trie = trie
if b.config.greedyScan {
b.trie = t
}
return nil
}

@ -25,6 +25,7 @@ type config struct {
maxKeySize int
maxValueSize int
sync bool
greedyScan bool
}
func (c *config) MarshalJSON() ([]byte, error) {
@ -33,11 +34,13 @@ func (c *config) MarshalJSON() ([]byte, error) {
MaxKeySize int `json:"max_key_size"`
MaxValueSize int `json:"max_value_size"`
Sync bool `json:"sync"`
GreedyScan bool `json:"greedy_scan"`
}{
MaxDatafileSize: c.maxDatafileSize,
MaxKeySize: c.maxKeySize,
MaxValueSize: c.maxValueSize,
Sync: c.sync,
GreedyScan: c.greedyScan,
})
}
@ -47,6 +50,7 @@ func getConfig(path string) (*config, error) {
MaxKeySize int `json:"max_key_size"`
MaxValueSize int `json:"max_value_size"`
Sync bool `json:"sync"`
GreedyScan bool `json:"greedy_scan"`
}
var cfg Config
@ -65,6 +69,7 @@ func getConfig(path string) (*config, error) {
maxKeySize: cfg.MaxKeySize,
maxValueSize: cfg.MaxValueSize,
sync: cfg.Sync,
greedyScan: cfg.GreedyScan,
}, nil
}
@ -108,3 +113,12 @@ func WithSync(sync bool) Option {
return nil
}
}
// WithGreedyScan enables faster Scan performance.
// This is disabled by default because it causes high memory usage.
func WithGreedyScan(enabled bool) Option {
return func(cfg *config) error {
cfg.greedyScan = enabled
return nil
}
}