From 0d6298995faa4c8772e33194de0d0c63115de21a Mon Sep 17 00:00:00 2001 From: "kayos@tcp.direct" Date: Mon, 27 Nov 2023 16:02:42 -0800 Subject: [PATCH] Fix: nil bitcask store during withnew when store exists --- bitcask/bitcask.go | 8 +- go.mod | 7 +- go.sum | 16 +- pogreb/LICENSE | 21 ++ pogreb/errors.go | 33 +++ pogreb/keeper_test.go | 30 +++ pogreb/pogreb.go | 306 ++++++++++++++++++++++++++++ pogreb/pogreb_search.go | 82 ++++++++ pogreb/pogreb_search_test.go | 265 ++++++++++++++++++++++++ pogreb/pogreb_test.go | 378 +++++++++++++++++++++++++++++++++++ 10 files changed, 1130 insertions(+), 16 deletions(-) create mode 100644 pogreb/LICENSE create mode 100644 pogreb/errors.go create mode 100644 pogreb/keeper_test.go create mode 100644 pogreb/pogreb.go create mode 100644 pogreb/pogreb_search.go create mode 100644 pogreb/pogreb_search_test.go create mode 100644 pogreb/pogreb_test.go diff --git a/bitcask/bitcask.go b/bitcask/bitcask.go index 5caabdf..7e678c6 100644 --- a/bitcask/bitcask.go +++ b/bitcask/bitcask.go @@ -138,11 +138,11 @@ func (db *DB) WithNew(storeName string, opts ...any) database.Filer { db.mu.RUnlock() err := db.Init(storeName) db.mu.RLock() - if err == nil { - return db.store[storeName] + if err != nil { + fmt.Println("error creating bitcask store: ", err) + } - fmt.Println("error creating bitcask store: ", err) - return Store{Bitcask: nil} + return db.store[storeName] } // Close is a simple shim for bitcask's Close function. diff --git a/go.mod b/go.mod index c96cffa..6be14c0 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.18 require ( git.tcp.direct/Mirrors/bitcask-mirror v0.0.0-20220228092422-1ec4297c7e34 - git.tcp.direct/kayos/common v0.8.6 + git.tcp.direct/kayos/common v0.9.3 + github.com/akrylysov/pogreb v0.10.1 github.com/davecgh/go-spew v1.1.1 - github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-multierror v1.0.0 ) require ( @@ -17,6 +18,6 @@ require ( github.com/plar/go-adaptive-radix-tree v1.0.4 // indirect github.com/rs/zerolog v1.26.1 // indirect golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85 // indirect - golang.org/x/sys v0.8.0 // indirect + golang.org/x/sys v0.13.0 // indirect nullprogram.com/x/rng v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 6fdbed2..d6f2877 100644 --- a/go.sum +++ b/go.sum @@ -39,15 +39,15 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.tcp.direct/Mirrors/bitcask-mirror v0.0.0-20220228092422-1ec4297c7e34 h1:tvoLGGLsQ0IYKKQPweMF5qRm3qO4gcTpuzi9jAr3Wkk= git.tcp.direct/Mirrors/bitcask-mirror v0.0.0-20220228092422-1ec4297c7e34/go.mod h1:NX/Gqm/iy6Tg2CfcmmB/kW/qzKKrGR6o2koOKA5KWnc= -git.tcp.direct/kayos/common v0.8.2 h1:Usev4zpFLRFGTjQ+5vhReYSS/3+XGOgYbVufIWqMMW8= -git.tcp.direct/kayos/common v0.8.2/go.mod h1:1XroljMDSTRybzyFGPTxs0gVb45YtH7wcehelU4koW8= -git.tcp.direct/kayos/common v0.8.6 h1:lt8nv+PrgAcbiOnbKUt7diza5hifR5fV3un6uIp/YVc= -git.tcp.direct/kayos/common v0.8.6/go.mod h1:QGGn7d2l4xBG7Cs/g84JzItPpHWjtfiyy+PSMnf6TzE= +git.tcp.direct/kayos/common v0.9.3 h1:MnQM4MH97zin+cloTsJRA3YEzagTGm/iR5sajpT+GoQ= +git.tcp.direct/kayos/common v0.9.3/go.mod h1:tTqUGj50mpwoQD0Zsdsv6cpDzN9VfjnQMgpDC8aRfms= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81 h1:uHogIJ9bXH75ZYrXnVShHIyywFiUZ7OOabwd9Sfd8rw= github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81/go.mod h1:6ZvnjTZX1LNo1oLpfaJK8h+MXqHxcBFBIwkgsv+xlv0= +github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= +github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -182,9 +182,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -488,9 +487,8 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pogreb/LICENSE b/pogreb/LICENSE new file mode 100644 index 0000000..0211f54 --- /dev/null +++ b/pogreb/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 kayos (kayos@tcp.direct) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pogreb/errors.go b/pogreb/errors.go new file mode 100644 index 0000000..610689b --- /dev/null +++ b/pogreb/errors.go @@ -0,0 +1,33 @@ +package pogreb + +import ( + "errors" + + "github.com/hashicorp/go-multierror" +) + +//goland:noinspection GoExportedElementShouldHaveComment +var ( + ErrUnknownAction = errors.New("unknown action") + ErrBogusStore = errors.New("bogus store backend") + ErrBadOptions = errors.New("invalid pogreb options") + ErrStoreExists = errors.New("store name already exists") + ErrNoStores = errors.New("no stores initialized") +) + +func namedErr(name string, err error) error { + if err == nil { + return nil + } + return multierror.Prefix(err, name) +} + +func compoundErrors(errs []error) (err error) { + for _, e := range errs { + if e == nil { + continue + } + err = multierror.Append(err, e) + } + return +} diff --git a/pogreb/keeper_test.go b/pogreb/keeper_test.go new file mode 100644 index 0000000..598902c --- /dev/null +++ b/pogreb/keeper_test.go @@ -0,0 +1,30 @@ +package pogreb + +import ( + "testing" + + "git.tcp.direct/tcp.direct/database" +) + +func Test_Interfaces(t *testing.T) { + v := OpenDB(t.TempDir()) + var keeper interface{} = v + if _, ok := keeper.(database.Keeper); !ok { + t.Error("Keeper interface not implemented") + } else { + t.Log("Keeper interface implemented") + } + vs := v.WithNew("test") + var searcher interface{} = vs + if _, ok := searcher.(database.Searcher); !ok { + t.Error("Searcher interface not implemented") + } else { + t.Log("Searcher interface implemented") + } + var filer interface{} = vs + if _, ok := filer.(database.Filer); !ok { + t.Error("Filer interface not implemented") + } else { + t.Log("Filer interface implemented") + } +} diff --git a/pogreb/pogreb.go b/pogreb/pogreb.go new file mode 100644 index 0000000..e0c20bd --- /dev/null +++ b/pogreb/pogreb.go @@ -0,0 +1,306 @@ +package pogreb + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + + "github.com/akrylysov/pogreb" + + "git.tcp.direct/tcp.direct/database" +) + +type Option func(*WrappedOptions) + +var OptionAllowRecovery = func(opts *WrappedOptions) { + opts.AllowRecovery = true +} + +func AllowRecovery() Option { + return OptionAllowRecovery +} + +func SetPogrebOptions(options pogreb.Options) Option { + return func(opts *WrappedOptions) { + opts.Options = &options + } +} + +type WrappedOptions struct { + *pogreb.Options + // AllowRecovery allows the database to be recovered if a lockfile is detected upon running Init. + AllowRecovery bool +} + +func (pstore *Store) Len() int { + return int(pstore.DB.Count()) +} + +func (pstore *Store) Keys() [][]byte { + iter := pstore.DB.Items() + ks := make([][]byte, pstore.DB.Count()) + for k, _, _ := iter.Next(); k != nil; k, _, _ = iter.Next() { + ks = append(ks, k) + } + return ks +} + +func (pstore *Store) Has(key []byte) bool { + ok, err := pstore.DB.Has(key) + if err != nil { + _, _ = os.Stderr.WriteString("error checking pogreb store for key: " + err.Error()) + } + return ok +} + +// Store is an implmentation of a Filer and a Searcher using Bitcask. +type Store struct { + *pogreb.DB + database.Searcher + opts *WrappedOptions + closed bool +} + +// Backend returns the underlying pogreb instance. +func (pstore *Store) Backend() any { + return pstore.DB +} + +// DB is a mapper of a Filer and Searcher implementation using pogreb. +type DB struct { + store map[string]database.Store + path string + mu *sync.RWMutex +} + +// AllStores returns a map of the names of all pogreb datastores and the corresponding Filers. +func (db *DB) AllStores() map[string]database.Filer { + db.mu.RLock() + defer db.mu.RUnlock() + var stores = make(map[string]database.Filer) + for n, s := range db.store { + stores[n] = s + } + return stores +} + +// FIXME: not returning the error is probably pretty irresponsible. + +// OpenDB will either open an existing set of pogreb datastores at the given directory, or it will create a new one. +func OpenDB(path string) *DB { + db := &DB{ + store: make(map[string]database.Store), + path: path, + mu: &sync.RWMutex{}, + } + if _, err := os.Stat(path); os.IsNotExist(err) { + err = os.MkdirAll(path, 0700) + if err != nil { + _, _ = os.Stderr.WriteString("error creating pogreb directory: " + err.Error()) + } + } + return db +} + +// Path returns the base path where we store our pogreb "stores". +func (db *DB) Path() string { + return db.path +} + +var defaultPogrebOptions = &WrappedOptions{ + Options: nil, + AllowRecovery: false, +} + +// SetDefaultPogrebOptions options will set the options used for all subsequent pogreb stores that are initialized. +func SetDefaultPogrebOptions(pogrebopts ...any) { + inner, pgoptOk := pogrebopts[0].(pogreb.Options) + wrapped, pgoptWrappedOk := pogrebopts[0].(*WrappedOptions) + switch { + case !pgoptOk && !pgoptWrappedOk: + panic("invalid pogreb options") + case pgoptOk: + defaultPogrebOptions = &WrappedOptions{ + Options: &inner, + AllowRecovery: false, + } + case pgoptWrappedOk: + defaultPogrebOptions = wrapped + } +} + +func normalizeOptions(opts ...any) *WrappedOptions { + var pogrebopts *WrappedOptions + pgInner, pgOK := opts[0].(pogreb.Options) + pgWrapped, pgWrappedOK := opts[0].(WrappedOptions) + switch { + case !pgOK && !pgWrappedOK: + return nil + case pgOK: + pogrebopts = &WrappedOptions{ + Options: &pgInner, + AllowRecovery: false, + } + case pgWrappedOK: + pogrebopts = &pgWrapped + } + return pogrebopts +} + +// Init opens a pogreb store at the given path to be referenced by storeName. +func (db *DB) Init(storeName string, opts ...any) error { + pogrebopts := defaultPogrebOptions + if len(opts) > 0 { + pogrebopts = normalizeOptions(opts...) + if pogrebopts == nil { + return ErrBadOptions + } + } + db.mu.Lock() + defer db.mu.Unlock() + if _, ok := db.store[storeName]; ok { + return ErrStoreExists + } + path := db.Path() + if _, err := os.Stat(filepath.Join(path, storeName, "lock")); !os.IsNotExist(err) && !pogrebopts.AllowRecovery { + return fmt.Errorf("%w: and seems to be running... "+ + "Please close it first, or use InitWithRecovery", ErrStoreExists) + } + c, e := pogreb.Open(filepath.Join(path, storeName), pogrebopts.Options) + if e != nil { + return e + } + db.store[storeName] = &Store{DB: c} + return nil +} + +// With calls the given underlying pogreb instance. +func (db *DB) With(storeName string) database.Store { + db.mu.RLock() + defer db.mu.RUnlock() + d, ok := db.store[storeName] + if ok { + return d + } + return nil +} + +// WithNew calls the given underlying pogreb instance, if it doesn't exist, it creates it. +func (db *DB) WithNew(storeName string, opts ...any) database.Filer { + pogrebopts := defaultPogrebOptions + if len(opts) > 0 { + if pogrebopts = normalizeOptions(opts...); pogrebopts == nil { + return nil + } + } + + db.mu.RLock() + defer db.mu.RUnlock() + + d, ok := db.store[storeName] + if ok { + return d + } + db.mu.RUnlock() + err := db.Init(storeName, pogrebopts) + db.mu.RLock() + if err == nil { + return db.store[storeName] + } + _, _ = os.Stderr.WriteString("error creating pogreb store: " + err.Error()) + return &Store{DB: nil} +} + +// Close is a simple shim for pogreb's Close function. +func (db *DB) Close(storeName string) error { + db.mu.Lock() + defer db.mu.Unlock() + st, ok := db.store[storeName] + if !ok { + return ErrBogusStore + } + err := st.Close() + if err != nil { + return err + } + delete(db.store, storeName) + return nil +} + +// Sync is a simple shim for pogreb's Sync function. +func (db *DB) Sync(storeName string) error { + db.mu.RLock() + defer db.mu.RUnlock() + return db.store[storeName].Backend().(*pogreb.DB).Sync() +} + +// withAllAction +type withAllAction uint8 + +const ( + // dclose + dclose withAllAction = iota + // dsync + dsync +) + +// withAll performs an action on all pogreb stores that we have open. +// In the case of an error, withAll will continue and return a compound form of any errors that occurred. +// For now this is just for Close and Sync, thusly it does a hard lock on the Keeper. +func (db *DB) withAll(action withAllAction) error { + if db == nil || db.store == nil || len(db.store) < 1 { + return ErrNoStores + } + var errs = make([]error, len(db.store)) + for name, store := range db.store { + var err error + if store == nil || store.Backend().(*pogreb.DB) == nil { + errs = append(errs, namedErr(name, ErrBogusStore)) + continue + } + switch action { + case dclose: + closeErr := store.Close() + if errors.Is(closeErr, fs.ErrClosed) { + continue + } + err = namedErr(name, closeErr) + case dsync: + err = namedErr(name, store.Sync()) + default: + return ErrUnknownAction + } + if err == nil { + continue + } + errs = append(errs, err) + } + return compoundErrors(errs) +} + +// SyncAndCloseAll implements the method from Keeper to sync and close all pogreb stores. +func (db *DB) SyncAndCloseAll() error { + var errs = make([]error, len(db.store)) + errSync := namedErr("sync", db.SyncAll()) + if errSync != nil { + errs = append(errs, errSync) + } + errClose := namedErr("close", db.CloseAll()) + if errClose != nil { + errs = append(errs, errClose) + } + return compoundErrors(errs) +} + +// CloseAll closes all pogreb datastores. +func (db *DB) CloseAll() error { + return db.withAll(dclose) +} + +// SyncAll syncs all pogreb datastores. +func (db *DB) SyncAll() error { + return db.withAll(dsync) +} diff --git a/pogreb/pogreb_search.go b/pogreb/pogreb_search.go new file mode 100644 index 0000000..2cce050 --- /dev/null +++ b/pogreb/pogreb_search.go @@ -0,0 +1,82 @@ +package pogreb + +import ( + "bytes" + "errors" + "strings" + + "github.com/akrylysov/pogreb" + + "git.tcp.direct/tcp.direct/database/kv" +) + +// Search will search for a given string within all values inside of a Store. +// Note, type casting will be necessary. (e.g: []byte or string) +func (pstore *Store) Search(query string) (<-chan *kv.KeyValue, chan error) { + var errChan = make(chan error) + var resChan = make(chan *kv.KeyValue, 5) + go func() { + defer func() { + close(resChan) + close(errChan) + }() + for _, key := range pstore.Keys() { + raw, err := pstore.Get(key) + if err != nil { + errChan <- err + continue + } + if raw != nil && strings.Contains(string(raw), query) { + keyVal := kv.NewKeyValue(kv.NewKey(key), kv.NewValue(raw)) + resChan <- keyVal + } + } + }() + return resChan, errChan +} + +// ValueExists will check for the existence of a Value anywhere within the keyspace; +// returning the first Key found, true if found || nil and false if not found. +func (pstore *Store) ValueExists(value []byte) (key []byte, ok bool) { + var raw []byte + var needle = kv.NewValue(value) + for _, key = range pstore.Keys() { + raw, _ = pstore.Get(key) + v := kv.NewValue(raw) + if v.Equal(needle) { + ok = true + return + } + } + return +} + +// PrefixScan will scan a Store for all keys that have a matching prefix of the given string +// and return a map of keys and values. (map[Key]Value) +// error channel will block, so be sure to read from it. +func (pstore *Store) PrefixScan(prefixs string) (<-chan *kv.KeyValue, chan error) { + prefix := []byte(prefixs) + errChan := make(chan error) + resChan := make(chan *kv.KeyValue, 5) + go func() { + var err error + defer func(e error) { + close(resChan) + close(errChan) + }(err) + iter := pstore.DB.Items() + for k, v, iterErr := iter.Next(); k != nil; k, v, iterErr = iter.Next() { + if errors.Is(iterErr, pogreb.ErrIterationDone) { + break + } + if iterErr != nil { + errChan <- iterErr + continue + } + if bytes.HasPrefix(k, prefix) { + resChan <- kv.NewKeyValue(kv.NewKey(k), kv.NewValue(v)) + } + } + }() + return resChan, errChan +} diff --git a/pogreb/pogreb_search_test.go b/pogreb/pogreb_search_test.go new file mode 100644 index 0000000..7552b66 --- /dev/null +++ b/pogreb/pogreb_search_test.go @@ -0,0 +1,265 @@ +package pogreb + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "testing" + + c "git.tcp.direct/kayos/common/entropy" + "github.com/davecgh/go-spew/spew" + + "git.tcp.direct/tcp.direct/database/kv" +) + +var needle = "yeet" + +type Foo struct { + Bar string + Yeet int + What map[string]int +} + +func setupTest(storename string, t *testing.T) *DB { + var db *DB + testpath := t.TempDir() + + t.Logf("opening database at %s", testpath) + db = OpenDB(testpath) + + err := db.Init(storename) + if err != nil { + t.Fatalf("[FAIL] couuldn't initialize store: %e", err) + } + + t.Cleanup(func() { + t.Logf("cleaning up file at; %s", testpath) + if err := os.RemoveAll(testpath); err != nil { + t.Error(err) + } + }) + return db +} + +func genJunk(t *testing.T, correct bool) []byte { + item := c.RandStr(5) + bar := c.RandStr(5) + if correct { + if c.RNG(100) > 50 { + item = needle + c.RandStr(c.RNG(5)) + } else { + bar = c.RandStr(c.RNG(5)) + needle + } + } + + f := Foo{ + Bar: bar, + Yeet: c.RNG(50), + What: map[string]int{c.RandStr(5): 2, item: 7}, + } + + raw, err := json.Marshal(f) + if err != nil { + t.Fatal(err.Error()) + } + return raw +} + +func addJunk(db *DB, storename string, one, two, three, four, five int, t *testing.T, echo bool) [][]byte { + var needles [][]byte + for n := 0; n != 100; n++ { + var rawjson []byte + switch n { + case one, two, three, four, five: + rawjson = genJunk(t, true) + needles = append(needles, rawjson) + default: + rawjson = genJunk(t, false) + } + err := db.With(storename).Put([]byte(fmt.Sprintf("%d", n)), rawjson) + if err != nil { + t.Fail() + t.Logf("%e", err) + } + } + if echo { + t.Logf( + "created 100 entries of random data with needles located at %d, %d, %d, %d, %d", + one, two, three, four, five, + ) + } else { + t.Log("created 100 entries of junk") + } + + return needles +} + +func Test_Search(t *testing.T) { + var storename = "test_search" + var db = setupTest(storename, t) + + one := c.RNG(100) + two := c.RNG(100) + three := c.RNG(100) + four := c.RNG(100) + five := c.RNG(100) + + addJunk(db, storename, one, two, three, four, five, t, true) + + // For coverage + db.store["yeet"] = &Store{DB: nil} + t.Run("BasicSearch", func(t *testing.T) { + t.Logf("executing search for %s", needle) + resChan, errChan := db.With(storename).Search(needle) + var keys = []int{one, two, three, four, five} + var needed = len(keys) + + for keyValue := range resChan { + keyint, err := strconv.Atoi(keyValue.Key.String()) + for _, k := range keys { + if keyint == k { + needed-- + } + } + keys = append(keys, keyint) + t.Logf("Found Key: %s, Value: %s", keyValue.Key.String(), keyValue.Value.String()) + + if err != nil { + t.Fatalf("failed to convert Key to int: %e", err) + } + select { + case err := <-errChan: + if err != nil { + t.Fatalf("failed to search: %e", err) + } + default: + continue + } + } + if needed != 0 { + t.Errorf("Needed %d results, got %d", len(keys), len(keys)-needed) + } + }) + + t.Run("NoResultsSearch", func(t *testing.T) { + bogus := c.RandStr(55) + t.Logf("executing search for %s", bogus) + var results []*kv.KeyValue + resChan, errChan := db.With(storename).Search(bogus) + select { + case err := <-errChan: + t.Errorf("failed to search: %s", err.Error()) + case r := <-resChan: + if r != nil { + spew.Dump(r) + results = append(results, r) + } + if len(results) > 0 { + t.Errorf("[FAIL] got %d results, wanted 0", len(results)) + } + } + }) +} + +func Test_ValueExists(t *testing.T) { + var storename = "test_value_exists" + var db = setupTest(storename, t) + + t.Run("ValueExists", func(t *testing.T) { + needles := addJunk(db, storename, c.RNG(100), c.RNG(100), c.RNG(100), c.RNG(100), c.RNG(100), t, true) + + for _, ndl := range needles { + k, exists := db.With(storename).ValueExists(ndl) + if !exists { + t.Fatalf("[FAIL] store should have contained a value %s somewhere, it did not.", string(ndl)) + } + if k == nil { + t.Fatalf("[FAIL] store should have contained a value %s somewhere, "+ + "it said it did but key was nil", string(ndl)) + } + v, _ := db.With(storename).Get(k) + if string(v) != string(ndl) { + t.Fatalf("[FAIL] retrieved value does not match search target %s != %s", string(v), string(ndl)) + } + t.Logf("[SUCCESS] successfully located value: %s, at key: %s", string(k), string(v)) + } + }) + + t.Run("ValueShouldNotExist", func(t *testing.T) { + for n := 0; n != 5; n++ { + garbage := c.RandStr(55) + if _, exists := db.With(storename).ValueExists([]byte(garbage)); exists { + t.Errorf("[FAIL] store should have not contained value %v, but it did", []byte(garbage)) + } else { + t.Logf("[SUCCESS] store succeeded in not having random value %s", garbage) + } + } + }) + + t.Run("ValueExistNilBitcask", func(t *testing.T) { + db.store["asdb"] = &Store{DB: nil} + garbage := "yeet" + if _, exists := db.With(storename).ValueExists([]byte(garbage)); exists { + t.Errorf("[FAIL] store should have not contained value %v, should have been nil", []byte(garbage)) + } else { + t.Log("[SUCCESS] store succeeded in being nil") + } + }) +} + +func Test_PrefixScan(t *testing.T) { + var storename = "test_prefix_scan" + var db = setupTest(storename, t) + addJunk(db, storename, c.RNG(5), c.RNG(5), c.RNG(5), c.RNG(5), c.RNG(5), t, false) + var needles = []*kv.KeyValue{ + kv.NewKeyValue(kv.NewKey([]byte("user:Frickhole")), kv.NewValue([]byte(c.RandStr(55)))), + kv.NewKeyValue(kv.NewKey([]byte("user:Johnson")), kv.NewValue([]byte(c.RandStr(55)))), + kv.NewKeyValue(kv.NewKey([]byte("user:Jackson")), kv.NewValue([]byte(c.RandStr(55)))), + kv.NewKeyValue(kv.NewKey([]byte("user:Frackhole")), kv.NewValue([]byte(c.RandStr(55)))), + kv.NewKeyValue(kv.NewKey([]byte("user:Baboshka")), kv.NewValue([]byte(c.RandStr(55)))), + } + for _, combo := range needles { + err := db.With(storename).Put(combo.Key.Bytes(), combo.Value.Bytes()) + if err != nil { + t.Fatalf("failed to add data to %s: %e", storename, err) + } else { + t.Logf("added needle with key(value): %s(%s)", combo.Key.String(), combo.Value.String()) + } + } + resChan, errChan := db.With(storename).PrefixScan("user:") + var results []*kv.KeyValue + for keyValue := range resChan { + results = append(results, keyValue) + select { + case err := <-errChan: + if err != nil { + t.Fatalf("failed to PrefixScan: %e", err) + } + break + default: + continue + } + } + if len(results) != len(needles) { + t.Errorf("[FAIL] Length of results (%d) is not the amount of needles we generated (%d)", len(results), len(needles)) + } + var keysmatched = 0 + for _, result := range results { + for _, ogkv := range needles { + if result.Key.String() != ogkv.Key.String() { + continue + } + t.Logf("Found needle key: %s", result.Key.String()) + keysmatched++ + if result.Value.String() != ogkv.Value.String() { + t.Errorf("[FAIL] values of key %s should have matched. wanted: %s, got: %s", + result.Key.String(), ogkv.Value.String(), result.Value.String()) + } + t.Logf("Found needle value: %s", ogkv.Value.String()) + } + } + if keysmatched != len(needles) { + t.Errorf("Needed to match %d keys, only matched %d", len(needles), len(needles)) + } +} diff --git a/pogreb/pogreb_test.go b/pogreb/pogreb_test.go new file mode 100644 index 0000000..f9e6ada --- /dev/null +++ b/pogreb/pogreb_test.go @@ -0,0 +1,378 @@ +package pogreb + +import ( + "bytes" + "errors" + "io/fs" + "os" + "path/filepath" + "testing" + + c "git.tcp.direct/kayos/common/entropy" + "github.com/davecgh/go-spew/spew" + + "git.tcp.direct/tcp.direct/database" +) + +func newTestDB(t *testing.T) database.Keeper { + t.Helper() + tpath := t.TempDir() + tdb := OpenDB(tpath) + if tdb == nil { + t.Fatalf("failed to open testdb at %s, got nil", tpath) + } + return tdb +} + +func seedRandKV(db database.Keeper, store string) error { + return db.With(store).Put([]byte(c.RandStr(55)), []byte(c.RandStr(55))) +} + +func seedRandStores(db database.Keeper, t *testing.T) { + t.Helper() + for n := 0; n != 5; n++ { + randstore := c.RandStr(5) + err := db.Init(randstore) + if err != nil { + t.Errorf("failed to initialize store for test SyncAndCloseAll: %e", err) + } + err = seedRandKV(db, randstore) + if err != nil { + t.Errorf("failed to initialize random values in store %s for test SyncAndCloseAll: %e", randstore, err) + } + } + t.Logf("seeded random stores with random values for test %s", t.Name()) +} + +func TestDB_Init(t *testing.T) { //nolint:funlen,gocognit,cyclop + var db = newTestDB(t) + type args struct{ storeName string } + type test struct { + name string + args args + wantErr bool + specErr error + } + tests := []test{ + { + name: "simple", + args: args{"simple"}, + wantErr: false, + }, + { + name: "storeExists", + args: args{"simple"}, + wantErr: true, + specErr: ErrStoreExists, + }, + { + name: "newStore", + args: args{"notsimple"}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := db.Init(tt.args.storeName) + if (err != nil) != tt.wantErr { + t.Errorf("[FAIL] Init() error = %v, wantErr %v", err, tt.wantErr) + } + if (err != nil) != tt.wantErr && tt.specErr != nil && !errors.Is(err, tt.specErr) { + t.Errorf("[FAIL] wanted error %e, got error %e", tt.specErr, err) + } + }) + } + + t.Run("withStoreTest", func(t *testing.T) { + key := []byte{51, 50} + value := []byte("string") + err := db.With("simple").Put(key, value) + t.Logf("Put Value %v at Key %v", string(value), key) + if err != nil { + t.Fatalf("[FAIL] %e", err) + } + gvalue, gerr := db.With("simple").Get(key) + if gerr != nil { + t.Fatalf("[FAIL] %e", gerr) + } + if !bytes.Equal(gvalue, value) { + t.Errorf("[FAIL] wanted %v, got %v", string(value), string(gvalue)) + } + t.Logf("Got Value %v at Key %v", string(gvalue), key) + }) + t.Run("withNewStoreDoesExist", func(t *testing.T) { + nope := db.WithNew("bing") + if err := nope.Put([]byte("key"), []byte("value")); err != nil { + t.Fatalf("[FAIL] %e", err) + } + err := nope.Put([]byte("bing"), []byte("bong")) + if err != nil { + t.Fatalf("[FAIL] %e", err) + } + yup := db.WithNew("bing") + res, err := yup.Get([]byte("bing")) + if err != nil { + t.Errorf("[FAIL] %e", err) + } + if !bytes.Equal(res, []byte("bong")) { + t.Errorf("[FAIL] wanted %v, got %v", string([]byte("bong")), string(res)) + } + }) + t.Run("withNewStoreDoesntExist", func(t *testing.T) { + if nope := db.WithNew("asdfqwerty"); nope.Backend() == nil { + t.Fatalf("[FAIL] got nil result for nonexistent store when it should have made itself: %T, %v", nope, nope) + } else { + t.Logf("[SUCCESS] got nil Value for store that doesn't exist") + } + }) + t.Run("withStoreDoesntExist", func(t *testing.T) { + nope := db.With(c.RandStr(10)) + if nope != nil { + t.Fatalf("[FAIL] got non nil result for nonexistent store: %T, %v", nope.Backend(), nope.Backend()) + } else { + t.Logf("[SUCCESS] got nil Value for store that doesn't exist") + } + }) + t.Run("syncAllShouldFail", func(t *testing.T) { + db.(*DB).store["wtf"] = &Store{} + t.Cleanup(func() { + t.Logf("deleting bogus store map entry") + delete(db.(*DB).store, "wtf") + }) + err := db.SyncAll() + if err == nil { + t.Fatalf("[FAIL] we should have gotten an error from bogus store map entry") + } + t.Logf("[SUCCESS] got compound error: %e", err) + }) + + // TODO: make sure sync is ACTUALLY sycing instead of only checking for nil err... ( ._. ) + + t.Run("syncAll", func(t *testing.T) { + err := db.SyncAll() + if err != nil { + t.Fatalf("[FAIL] got compound error: %e", err) + } + }) + t.Run("closeAll", func(t *testing.T) { + t.Cleanup(func() { + err := os.RemoveAll("./testdata") + if err != nil { + t.Fatalf("[CLEANUP FAIL] %e", err) + } + t.Logf("[CLEANUP] cleaned up ./testdata") + }) + err := db.CloseAll() + if err != nil { + t.Fatalf("[FAIL] got compound error: %e", err) + } + db = nil + }) + t.Run("SyncAndCloseAll", func(t *testing.T) { + db = newTestDB(t) + seedRandStores(db, t) + err := db.SyncAndCloseAll() + if err != nil { + t.Errorf("[FAIL] failed to SyncAndCloseAll: %e", err) + } + }) +} + +func Test_Sync(t *testing.T) { + // TODO: make sure sync is ACTUALLY sycing instead of only checking for nil err... + var db = newTestDB(t) + seedRandStores(db, t) + t.Run("Sync", func(t *testing.T) { + for d := range db.(*DB).store { + err := db.With(d).Sync() + if err != nil { + t.Errorf("[FAIL] failed to sync %s: %e", d, err) + } else { + t.Logf("[+] Sync() successful for %s", d) + } + } + }) +} + +func Test_Close(t *testing.T) { + var db = newTestDB(t) + defer func() { + db = nil + }() + seedRandStores(db, t) + var oldstores []string + t.Run("Close", func(t *testing.T) { + for d := range db.AllStores() { + oldstores = append(oldstores, d) + err := db.Close(d) + if err != nil { + t.Fatalf("[FAIL] failed to close %s: %v", d, err) + } + t.Logf("[+] Close() successful for %s", d) + } + t.Run("AssureClosed", func(t *testing.T) { + for _, d := range oldstores { + if st := db.With(d); st != nil { + spew.Dump(st) + t.Errorf("[FAIL] store %s should have been deleted", d) + } + } + t.Logf("[SUCCESS] Confirmed that all stores have been closed") + }) + }) + + t.Run("CantCloseBogusStore", func(t *testing.T) { + err := db.Close(c.RandStr(55)) + if !errors.Is(err, ErrBogusStore) { + t.Errorf("[FAIL] got err %e, wanted err %e", err, ErrBogusStore) + } + }) +} + +func Test_withAll(t *testing.T) { + var db = newTestDB(t) + asdf1 := c.RandStr(10) + asdf2 := c.RandStr(10) + + defer func() { + if err := db.CloseAll(); err != nil && !errors.Is(err, fs.ErrClosed) { + t.Errorf("[FAIL] failed to close all stores: %v", err) + } + }() + t.Run("withAllNoStores", func(t *testing.T) { + err := db.(*DB).withAll(121) + if !errors.Is(err, ErrNoStores) { + t.Errorf("[FAIL] got err %e, wanted err %e", err, ErrNoStores) + } + }) + t.Run("withAllNilMap", func(t *testing.T) { + nilDb := newTestDB(t) + nilDb.(*DB).store = nil + err := nilDb.(*DB).withAll(dclose) + if err == nil { + t.Errorf("[FAIL] got nil err from trying to work on nil map, wanted err") + } + }) + t.Run("withAllBogusAction", func(t *testing.T) { + err := db.Init(asdf1) + if err != nil { + t.Errorf("[FAIL] unexpected error: %e", err) + } + wAllErr := db.(*DB).withAll(121) + if !errors.Is(wAllErr, ErrUnknownAction) { + t.Errorf("[FAIL] wanted error %e, got error %e", ErrUnknownAction, err) + } + }) + t.Run("ListAll", func(t *testing.T) { + allStores := db.AllStores() + if len(allStores) == 0 { + t.Errorf("[FAIL] no stores found") + } + for n, s := range allStores { + if n == "" { + t.Errorf("[FAIL] store name is empty") + } + if s == nil { + t.Errorf("[FAIL] store is nil") + } + t.Logf("[+] found store named %s: %v", n, s) + } + if len(allStores) != len(db.(*DB).store) { + t.Errorf("[FAIL] found %d stores, expected %d", len(allStores), len(db.(*DB).store)) + } + }) + t.Run("ListAllAndInteract", func(t *testing.T) { + err := db.Init(asdf2) + if err != nil { + t.Errorf("[FAIL] unexpected error: %e", err) + } + err = db.With(asdf1).Put([]byte("asdf"), []byte("asdf")) + if err != nil { + t.Errorf("[FAIL] unexpected error: %e", err) + } + err = db.With(asdf2).Put([]byte("asdf"), []byte("asdf")) + if err != nil { + t.Errorf("[FAIL] unexpected error: %e", err) + } + allStores := db.AllStores() + if len(allStores) == 0 { + t.Errorf("[FAIL] no stores found") + } + for n, s := range allStores { + if n == "" { + t.Errorf("[FAIL] store name is empty") + } + if s == nil { + t.Errorf("[FAIL] store is nil") + } + if len(db.(*DB).store) != 2 { + t.Errorf("[SANITY FAIL] found %d stores, expected %d", len(allStores), len(db.(*DB).store)) + } + t.Logf("[+] found store named %s: %v", n, s) + if len(allStores) != len(db.(*DB).store) { + t.Errorf("[FAIL] found %d stores, expected %d", len(allStores), len(db.(*DB).store)) + } + var res []byte + res, err = db.With(n).Get([]byte("asdf")) + if err != nil { + t.Errorf("[FAIL] unexpected error: %v", err) + } + if !bytes.Equal(res, []byte("asdf")) { + t.Errorf("[FAIL] expected %s, got %s", n, res) + } else { + t.Logf("[+] found %s in store %s", res, n) + } + } + }) + t.Run("WithAllIncludingBadStore", func(t *testing.T) { + db.(*DB).store["yeeterson"] = &Store{} + err := db.(*DB).withAll(dclose) + if err == nil { + t.Errorf("[FAIL] got nil err, wanted any error") + } + delete(db.(*DB).store, "yeeterson") + }) +} + +func Test_WithOptions(t *testing.T) { //nolint:funlen,gocognit,cyclop + tpath := t.TempDir() + tdb := OpenDB(tpath) + if tdb == nil { + t.Fatalf("failed to open testdb at %s, got nil", tpath) + } + // FIXME: inconsistent with other implementations (bitcask) + defer func() { + t.Helper() + err := tdb.CloseAll() + if err == nil { + t.Fatalf("[FAIL] was able to close uninitialized store, expected error") + } + }() + t.Run("InitWithBogusOption", func(t *testing.T) { + db := newTestDB(t) + err := db.Init("bogus", "yeet") + if err == nil { + t.Errorf("[FAIL] Init should have failed with bogus option") + } + }) +} +func Test_PhonyInit(t *testing.T) { + newtmp := t.TempDir() + err := os.MkdirAll(newtmp+"/"+t.Name(), 0755) + if err != nil { + t.Fatalf("[FAIL] failed to create test directory: %e", err) + } + err = os.Symlink("/dev/null", filepath.Join(newtmp, t.Name(), "lock")) + if err != nil { + t.Fatal(err.Error()) + } + tdb := OpenDB(newtmp) + defer func() { + _ = tdb.CloseAll() + }() + err = tdb.Init(t.Name()) + if err == nil { + t.Error("[FAIL] expected error while trying to open a store where lock exists, got nil") + } +}