tcp.direct
/
database
16
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix: nil bitcask store during withnew when store exists

This commit is contained in:
kayos@tcp.direct 2023-11-27 16:02:42 -08:00
parent c2d7ae8d3e
commit 0d6298995f
Signed by: kayos
GPG Key ID: 4B841471B4BEE979
10 changed files with 1130 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bitcask/bitcask.go
View File

@ -138,11 +138,11 @@ func (db *DB) WithNew(storeName string, opts ...any) database.Filer {
db.mu.RUnlock()
err := db.Init(storeName)
db.mu.RLock()
if err == nil {
return db.store[storeName]
if err != nil {
fmt.Println("error creating bitcask store: ", err)
}
fmt.Println("error creating bitcask store: ", err)
return Store{Bitcask: nil}
return db.store[storeName]
}
// Close is a simple shim for bitcask's Close function.

7
go.mod
View File

@ -4,9 +4,10 @@ go 1.18
require (
git.tcp.direct/Mirrors/bitcask-mirror v0.0.0-20220228092422-1ec4297c7e34
git.tcp.direct/kayos/common v0.8.6
git.tcp.direct/kayos/common v0.9.3
github.com/akrylysov/pogreb v0.10.1
github.com/davecgh/go-spew v1.1.1
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-multierror v1.0.0
)
require (
@ -17,6 +18,6 @@ require (
github.com/plar/go-adaptive-radix-tree v1.0.4 // indirect
github.com/rs/zerolog v1.26.1 // indirect
golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/sys v0.13.0 // indirect
nullprogram.com/x/rng v1.1.0 // indirect
)

16
go.sum
View File

@ -39,15 +39,15 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.tcp.direct/Mirrors/bitcask-mirror v0.0.0-20220228092422-1ec4297c7e34 h1:tvoLGGLsQ0IYKKQPweMF5qRm3qO4gcTpuzi9jAr3Wkk=
git.tcp.direct/Mirrors/bitcask-mirror v0.0.0-20220228092422-1ec4297c7e34/go.mod h1:NX/Gqm/iy6Tg2CfcmmB/kW/qzKKrGR6o2koOKA5KWnc=
git.tcp.direct/kayos/common v0.8.2 h1:Usev4zpFLRFGTjQ+5vhReYSS/3+XGOgYbVufIWqMMW8=
git.tcp.direct/kayos/common v0.8.2/go.mod h1:1XroljMDSTRybzyFGPTxs0gVb45YtH7wcehelU4koW8=
git.tcp.direct/kayos/common v0.8.6 h1:lt8nv+PrgAcbiOnbKUt7diza5hifR5fV3un6uIp/YVc=
git.tcp.direct/kayos/common v0.8.6/go.mod h1:QGGn7d2l4xBG7Cs/g84JzItPpHWjtfiyy+PSMnf6TzE=
git.tcp.direct/kayos/common v0.9.3 h1:MnQM4MH97zin+cloTsJRA3YEzagTGm/iR5sajpT+GoQ=
git.tcp.direct/kayos/common v0.9.3/go.mod h1:tTqUGj50mpwoQD0Zsdsv6cpDzN9VfjnQMgpDC8aRfms=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81 h1:uHogIJ9bXH75ZYrXnVShHIyywFiUZ7OOabwd9Sfd8rw=
github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81/go.mod h1:6ZvnjTZX1LNo1oLpfaJK8h+MXqHxcBFBIwkgsv+xlv0=
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@ -182,9 +182,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@ -488,9 +487,8 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

21
pogreb/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 kayos (kayos@tcp.direct)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

33
pogreb/errors.go Normal file
View File

@ -0,0 +1,33 @@
package pogreb
import (
"errors"
"github.com/hashicorp/go-multierror"
)
//goland:noinspection GoExportedElementShouldHaveComment
var (
ErrUnknownAction = errors.New("unknown action")
ErrBogusStore = errors.New("bogus store backend")
ErrBadOptions = errors.New("invalid pogreb options")
ErrStoreExists = errors.New("store name already exists")
ErrNoStores = errors.New("no stores initialized")
)
func namedErr(name string, err error) error {
if err == nil {
return nil
}
return multierror.Prefix(err, name)
}
func compoundErrors(errs []error) (err error) {
for _, e := range errs {
if e == nil {
continue
}
err = multierror.Append(err, e)
}
return
}

30
pogreb/keeper_test.go Normal file
View File

@ -0,0 +1,30 @@
package pogreb
import (
"testing"
"git.tcp.direct/tcp.direct/database"
)
func Test_Interfaces(t *testing.T) {
v := OpenDB(t.TempDir())
var keeper interface{} = v
if _, ok := keeper.(database.Keeper); !ok {
t.Error("Keeper interface not implemented")
} else {
t.Log("Keeper interface implemented")
}
vs := v.WithNew("test")
var searcher interface{} = vs
if _, ok := searcher.(database.Searcher); !ok {
t.Error("Searcher interface not implemented")
} else {
t.Log("Searcher interface implemented")
}
var filer interface{} = vs
if _, ok := filer.(database.Filer); !ok {
t.Error("Filer interface not implemented")
} else {
t.Log("Filer interface implemented")
}
}

306
pogreb/pogreb.go Normal file
View File

@ -0,0 +1,306 @@
package pogreb
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"github.com/akrylysov/pogreb"
"git.tcp.direct/tcp.direct/database"
)
type Option func(*WrappedOptions)
var OptionAllowRecovery = func(opts *WrappedOptions) {
opts.AllowRecovery = true
}
func AllowRecovery() Option {
return OptionAllowRecovery
}
func SetPogrebOptions(options pogreb.Options) Option {
return func(opts *WrappedOptions) {
opts.Options = &options
}
}
type WrappedOptions struct {
*pogreb.Options
// AllowRecovery allows the database to be recovered if a lockfile is detected upon running Init.
AllowRecovery bool
}
func (pstore *Store) Len() int {
return int(pstore.DB.Count())
}
func (pstore *Store) Keys() [][]byte {
iter := pstore.DB.Items()
ks := make([][]byte, pstore.DB.Count())
for k, _, _ := iter.Next(); k != nil; k, _, _ = iter.Next() {
ks = append(ks, k)
}
return ks
}
func (pstore *Store) Has(key []byte) bool {
ok, err := pstore.DB.Has(key)
if err != nil {
_, _ = os.Stderr.WriteString("error checking pogreb store for key: " + err.Error())
}
return ok
}
// Store is an implmentation of a Filer and a Searcher using Bitcask.
type Store struct {
*pogreb.DB
database.Searcher
opts *WrappedOptions
closed bool
}
// Backend returns the underlying pogreb instance.
func (pstore *Store) Backend() any {
return pstore.DB
}
// DB is a mapper of a Filer and Searcher implementation using pogreb.
type DB struct {
store map[string]database.Store
path string
mu *sync.RWMutex
}
// AllStores returns a map of the names of all pogreb datastores and the corresponding Filers.
func (db *DB) AllStores() map[string]database.Filer {
db.mu.RLock()
defer db.mu.RUnlock()
var stores = make(map[string]database.Filer)
for n, s := range db.store {
stores[n] = s
}
return stores
}
// FIXME: not returning the error is probably pretty irresponsible.
// OpenDB will either open an existing set of pogreb datastores at the given directory, or it will create a new one.
func OpenDB(path string) *DB {
db := &DB{
store: make(map[string]database.Store),
path: path,
mu: &sync.RWMutex{},
}
if _, err := os.Stat(path); os.IsNotExist(err) {
err = os.MkdirAll(path, 0700)
if err != nil {
_, _ = os.Stderr.WriteString("error creating pogreb directory: " + err.Error())
}
}
return db
}
// Path returns the base path where we store our pogreb "stores".
func (db *DB) Path() string {
return db.path
}
var defaultPogrebOptions = &WrappedOptions{
Options: nil,
AllowRecovery: false,
}
// SetDefaultPogrebOptions options will set the options used for all subsequent pogreb stores that are initialized.
func SetDefaultPogrebOptions(pogrebopts ...any) {
inner, pgoptOk := pogrebopts[0].(pogreb.Options)
wrapped, pgoptWrappedOk := pogrebopts[0].(*WrappedOptions)
switch {
case !pgoptOk && !pgoptWrappedOk:
panic("invalid pogreb options")
case pgoptOk:
defaultPogrebOptions = &WrappedOptions{
Options: &inner,
AllowRecovery: false,
}
case pgoptWrappedOk:
defaultPogrebOptions = wrapped
}
}
func normalizeOptions(opts ...any) *WrappedOptions {
var pogrebopts *WrappedOptions
pgInner, pgOK := opts[0].(pogreb.Options)
pgWrapped, pgWrappedOK := opts[0].(WrappedOptions)
switch {
case !pgOK && !pgWrappedOK:
return nil
case pgOK:
pogrebopts = &WrappedOptions{
Options: &pgInner,
AllowRecovery: false,
}
case pgWrappedOK:
pogrebopts = &pgWrapped
}
return pogrebopts
}
// Init opens a pogreb store at the given path to be referenced by storeName.
func (db *DB) Init(storeName string, opts ...any) error {
pogrebopts := defaultPogrebOptions
if len(opts) > 0 {
pogrebopts = normalizeOptions(opts...)
if pogrebopts == nil {
return ErrBadOptions
}
}
db.mu.Lock()
defer db.mu.Unlock()
if _, ok := db.store[storeName]; ok {
return ErrStoreExists
}
path := db.Path()
if _, err := os.Stat(filepath.Join(path, storeName, "lock")); !os.IsNotExist(err) && !pogrebopts.AllowRecovery {
return fmt.Errorf("%w: and seems to be running... "+
"Please close it first, or use InitWithRecovery", ErrStoreExists)
}
c, e := pogreb.Open(filepath.Join(path, storeName), pogrebopts.Options)
if e != nil {
return e
}
db.store[storeName] = &Store{DB: c}
return nil
}
// With calls the given underlying pogreb instance.
func (db *DB) With(storeName string) database.Store {
db.mu.RLock()
defer db.mu.RUnlock()
d, ok := db.store[storeName]
if ok {
return d
}
return nil
}
// WithNew calls the given underlying pogreb instance, if it doesn't exist, it creates it.
func (db *DB) WithNew(storeName string, opts ...any) database.Filer {
pogrebopts := defaultPogrebOptions
if len(opts) > 0 {
if pogrebopts = normalizeOptions(opts...); pogrebopts == nil {
return nil
}
}
db.mu.RLock()
defer db.mu.RUnlock()
d, ok := db.store[storeName]
if ok {
return d
}
db.mu.RUnlock()
err := db.Init(storeName, pogrebopts)
db.mu.RLock()
if err == nil {
return db.store[storeName]
}
_, _ = os.Stderr.WriteString("error creating pogreb store: " + err.Error())
return &Store{DB: nil}
}
// Close is a simple shim for pogreb's Close function.
func (db *DB) Close(storeName string) error {
db.mu.Lock()
defer db.mu.Unlock()
st, ok := db.store[storeName]
if !ok {
return ErrBogusStore
}
err := st.Close()
if err != nil {
return err
}
delete(db.store, storeName)
return nil
}
// Sync is a simple shim for pogreb's Sync function.
func (db *DB) Sync(storeName string) error {
db.mu.RLock()
defer db.mu.RUnlock()
return db.store[storeName].Backend().(*pogreb.DB).Sync()
}
// withAllAction
type withAllAction uint8
const (
// dclose
dclose withAllAction = iota
// dsync
dsync
)
// withAll performs an action on all pogreb stores that we have open.
// In the case of an error, withAll will continue and return a compound form of any errors that occurred.
// For now this is just for Close and Sync, thusly it does a hard lock on the Keeper.
func (db *DB) withAll(action withAllAction) error {
if db == nil || db.store == nil || len(db.store) < 1 {
return ErrNoStores
}
var errs = make([]error, len(db.store))
for name, store := range db.store {
var err error
if store == nil || store.Backend().(*pogreb.DB) == nil {
errs = append(errs, namedErr(name, ErrBogusStore))
continue
}
switch action {
case dclose:
closeErr := store.Close()
if errors.Is(closeErr, fs.ErrClosed) {
continue
}
err = namedErr(name, closeErr)
case dsync:
err = namedErr(name, store.Sync())
default:
return ErrUnknownAction
}
if err == nil {
continue
}
errs = append(errs, err)
}
return compoundErrors(errs)
}
// SyncAndCloseAll implements the method from Keeper to sync and close all pogreb stores.
func (db *DB) SyncAndCloseAll() error {
var errs = make([]error, len(db.store))
errSync := namedErr("sync", db.SyncAll())
if errSync != nil {
errs = append(errs, errSync)
}
errClose := namedErr("close", db.CloseAll())
if errClose != nil {
errs = append(errs, errClose)
}
return compoundErrors(errs)
}
// CloseAll closes all pogreb datastores.
func (db *DB) CloseAll() error {
return db.withAll(dclose)
}
// SyncAll syncs all pogreb datastores.
func (db *DB) SyncAll() error {
return db.withAll(dsync)
}

82
pogreb/pogreb_search.go Normal file
View File

@ -0,0 +1,82 @@
package pogreb
import (
"bytes"
"errors"
"strings"
"github.com/akrylysov/pogreb"
"git.tcp.direct/tcp.direct/database/kv"
)
// Search will search for a given string within all values inside of a Store.
// Note, type casting will be necessary. (e.g: []byte or string)
func (pstore *Store) Search(query string) (<-chan *kv.KeyValue, chan error) {
var errChan = make(chan error)
var resChan = make(chan *kv.KeyValue, 5)
go func() {
defer func() {
close(resChan)
close(errChan)
}()
for _, key := range pstore.Keys() {
raw, err := pstore.Get(key)
if err != nil {
errChan <- err
continue
}
if raw != nil && strings.Contains(string(raw), query) {
keyVal := kv.NewKeyValue(kv.NewKey(key), kv.NewValue(raw))
resChan <- keyVal
}
}
}()
return resChan, errChan
}
// ValueExists will check for the existence of a Value anywhere within the keyspace;
// returning the first Key found, true if found || nil and false if not found.
func (pstore *Store) ValueExists(value []byte) (key []byte, ok bool) {
var raw []byte
var needle = kv.NewValue(value)
for _, key = range pstore.Keys() {
raw, _ = pstore.Get(key)
v := kv.NewValue(raw)
if v.Equal(needle) {
ok = true
return
}
}
return
}
// PrefixScan will scan a Store for all keys that have a matching prefix of the given string
// and return a map of keys and values. (map[Key]Value)
// error channel will block, so be sure to read from it.
func (pstore *Store) PrefixScan(prefixs string) (<-chan *kv.KeyValue, chan error) {
prefix := []byte(prefixs)
errChan := make(chan error)
resChan := make(chan *kv.KeyValue, 5)
go func() {
var err error
defer func(e error) {
close(resChan)
close(errChan)
}(err)
iter := pstore.DB.Items()
for k, v, iterErr := iter.Next(); k != nil; k, v, iterErr = iter.Next() {
if errors.Is(iterErr, pogreb.ErrIterationDone) {
break
}
if iterErr != nil {
errChan <- iterErr
continue
}
if bytes.HasPrefix(k, prefix) {
resChan <- kv.NewKeyValue(kv.NewKey(k), kv.NewValue(v))
}
}
}()
return resChan, errChan
}

265
pogreb/pogreb_search_test.go Normal file
View File

@ -0,0 +1,265 @@
package pogreb
import (
"encoding/json"
"fmt"
"os"
"strconv"
"testing"
c "git.tcp.direct/kayos/common/entropy"
"github.com/davecgh/go-spew/spew"
"git.tcp.direct/tcp.direct/database/kv"
)
var needle = "yeet"
type Foo struct {
Bar string
Yeet int
What map[string]int
}
func setupTest(storename string, t *testing.T) *DB {
var db *DB
testpath := t.TempDir()
t.Logf("opening database at %s", testpath)
db = OpenDB(testpath)
err := db.Init(storename)
if err != nil {
t.Fatalf("[FAIL] couuldn't initialize store: %e", err)
}
t.Cleanup(func() {
t.Logf("cleaning up file at; %s", testpath)
if err := os.RemoveAll(testpath); err != nil {
t.Error(err)
}
})
return db
}
func genJunk(t *testing.T, correct bool) []byte {
item := c.RandStr(5)
bar := c.RandStr(5)
if correct {
if c.RNG(100) > 50 {
item = needle + c.RandStr(c.RNG(5))
} else {
bar = c.RandStr(c.RNG(5)) + needle
}
}
f := Foo{
Bar: bar,
Yeet: c.RNG(50),
What: map[string]int{c.RandStr(5): 2, item: 7},
}
raw, err := json.Marshal(f)
if err != nil {
t.Fatal(err.Error())
}
return raw
}
func addJunk(db *DB, storename string, one, two, three, four, five int, t *testing.T, echo bool) [][]byte {
var needles [][]byte
for n := 0; n != 100; n++ {
var rawjson []byte
switch n {
case one, two, three, four, five:
rawjson = genJunk(t, true)
needles = append(needles, rawjson)
default:
rawjson = genJunk(t, false)
}
err := db.With(storename).Put([]byte(fmt.Sprintf("%d", n)), rawjson)
if err != nil {
t.Fail()
t.Logf("%e", err)
}
}
if echo {
t.Logf(
"created 100 entries of random data with needles located at %d, %d, %d, %d, %d",
one, two, three, four, five,
)
} else {
t.Log("created 100 entries of junk")
}
return needles
}
func Test_Search(t *testing.T) {
var storename = "test_search"
var db = setupTest(storename, t)
one := c.RNG(100)
two := c.RNG(100)
three := c.RNG(100)
four := c.RNG(100)
five := c.RNG(100)
addJunk(db, storename, one, two, three, four, five, t, true)
// For coverage
db.store["yeet"] = &Store{DB: nil}
t.Run("BasicSearch", func(t *testing.T) {
t.Logf("executing search for %s", needle)
resChan, errChan := db.With(storename).Search(needle)
var keys = []int{one, two, three, four, five}
var needed = len(keys)
for keyValue := range resChan {
keyint, err := strconv.Atoi(keyValue.Key.String())
for _, k := range keys {
if keyint == k {
needed--
}
}
keys = append(keys, keyint)
t.Logf("Found Key: %s, Value: %s", keyValue.Key.String(), keyValue.Value.String())
if err != nil {
t.Fatalf("failed to convert Key to int: %e", err)
}
select {
case err := <-errChan:
if err != nil {
t.Fatalf("failed to search: %e", err)
}
default:
continue
}
}
if needed != 0 {
t.Errorf("Needed %d results, got %d", len(keys), len(keys)-needed)
}
})
t.Run("NoResultsSearch", func(t *testing.T) {
bogus := c.RandStr(55)
t.Logf("executing search for %s", bogus)
var results []*kv.KeyValue
resChan, errChan := db.With(storename).Search(bogus)
select {
case err := <-errChan:
t.Errorf("failed to search: %s", err.Error())
case r := <-resChan:
if r != nil {
spew.Dump(r)
results = append(results, r)
}
if len(results) > 0 {
t.Errorf("[FAIL] got %d results, wanted 0", len(results))
}
}
})
}
func Test_ValueExists(t *testing.T) {
var storename = "test_value_exists"
var db = setupTest(storename, t)
t.Run("ValueExists", func(t *testing.T) {
needles := addJunk(db, storename, c.RNG(100), c.RNG(100), c.RNG(100), c.RNG(100), c.RNG(100), t, true)
for _, ndl := range needles {
k, exists := db.With(storename).ValueExists(ndl)
if !exists {
t.Fatalf("[FAIL] store should have contained a value %s somewhere, it did not.", string(ndl))
}
if k == nil {
t.Fatalf("[FAIL] store should have contained a value %s somewhere, "+
"it said it did but key was nil", string(ndl))
}
v, _ := db.With(storename).Get(k)
if string(v) != string(ndl) {
t.Fatalf("[FAIL] retrieved value does not match search target %s != %s", string(v), string(ndl))
}
t.Logf("[SUCCESS] successfully located value: %s, at key: %s", string(k), string(v))
}
})
t.Run("ValueShouldNotExist", func(t *testing.T) {
for n := 0; n != 5; n++ {
garbage := c.RandStr(55)
if _, exists := db.With(storename).ValueExists([]byte(garbage)); exists {
t.Errorf("[FAIL] store should have not contained value %v, but it did", []byte(garbage))
} else {
t.Logf("[SUCCESS] store succeeded in not having random value %s", garbage)
}
}
})
t.Run("ValueExistNilBitcask", func(t *testing.T) {
db.store["asdb"] = &Store{DB: nil}
garbage := "yeet"
if _, exists := db.With(storename).ValueExists([]byte(garbage)); exists {
t.Errorf("[FAIL] store should have not contained value %v, should have been nil", []byte(garbage))
} else {
t.Log("[SUCCESS] store succeeded in being nil")
}
})
}
func Test_PrefixScan(t *testing.T) {
var storename = "test_prefix_scan"
var db = setupTest(storename, t)
addJunk(db, storename, c.RNG(5), c.RNG(5), c.RNG(5), c.RNG(5), c.RNG(5), t, false)
var needles = []*kv.KeyValue{
kv.NewKeyValue(kv.NewKey([]byte("user:Frickhole")), kv.NewValue([]byte(c.RandStr(55)))),
kv.NewKeyValue(kv.NewKey([]byte("user:Johnson")), kv.NewValue([]byte(c.RandStr(55)))),
kv.NewKeyValue(kv.NewKey([]byte("user:Jackson")), kv.NewValue([]byte(c.RandStr(55)))),
kv.NewKeyValue(kv.NewKey([]byte("user:Frackhole")), kv.NewValue([]byte(c.RandStr(55)))),
kv.NewKeyValue(kv.NewKey([]byte("user:Baboshka")), kv.NewValue([]byte(c.RandStr(55)))),
}
for _, combo := range needles {
err := db.With(storename).Put(combo.Key.Bytes(), combo.Value.Bytes())
if err != nil {
t.Fatalf("failed to add data to %s: %e", storename, err)
} else {
t.Logf("added needle with key(value): %s(%s)", combo.Key.String(), combo.Value.String())
}
}
resChan, errChan := db.With(storename).PrefixScan("user:")
var results []*kv.KeyValue
for keyValue := range resChan {
results = append(results, keyValue)
select {
case err := <-errChan:
if err != nil {
t.Fatalf("failed to PrefixScan: %e", err)
}
break
default:
continue
}
}
if len(results) != len(needles) {
t.Errorf("[FAIL] Length of results (%d) is not the amount of needles we generated (%d)", len(results), len(needles))
}
var keysmatched = 0
for _, result := range results {
for _, ogkv := range needles {
if result.Key.String() != ogkv.Key.String() {
continue
}
t.Logf("Found needle key: %s", result.Key.String())
keysmatched++
if result.Value.String() != ogkv.Value.String() {
t.Errorf("[FAIL] values of key %s should have matched. wanted: %s, got: %s",
result.Key.String(), ogkv.Value.String(), result.Value.String())
}
t.Logf("Found needle value: %s", ogkv.Value.String())
}
}
if keysmatched != len(needles) {
t.Errorf("Needed to match %d keys, only matched %d", len(needles), len(needles))
}
}

378
pogreb/pogreb_test.go Normal file
View File

@ -0,0 +1,378 @@
package pogreb
import (
"bytes"
"errors"
"io/fs"
"os"
"path/filepath"
"testing"
c "git.tcp.direct/kayos/common/entropy"
"github.com/davecgh/go-spew/spew"
"git.tcp.direct/tcp.direct/database"
)
func newTestDB(t *testing.T) database.Keeper {
t.Helper()
tpath := t.TempDir()
tdb := OpenDB(tpath)
if tdb == nil {
t.Fatalf("failed to open testdb at %s, got nil", tpath)
}
return tdb
}
func seedRandKV(db database.Keeper, store string) error {
return db.With(store).Put([]byte(c.RandStr(55)), []byte(c.RandStr(55)))
}
func seedRandStores(db database.Keeper, t *testing.T) {
t.Helper()
for n := 0; n != 5; n++ {
randstore := c.RandStr(5)
err := db.Init(randstore)
if err != nil {
t.Errorf("failed to initialize store for test SyncAndCloseAll: %e", err)
}
err = seedRandKV(db, randstore)
if err != nil {
t.Errorf("failed to initialize random values in store %s for test SyncAndCloseAll: %e", randstore, err)
}
}
t.Logf("seeded random stores with random values for test %s", t.Name())
}
func TestDB_Init(t *testing.T) { //nolint:funlen,gocognit,cyclop
var db = newTestDB(t)
type args struct{ storeName string }
type test struct {
name string
args args
wantErr bool
specErr error
}
tests := []test{
{
name: "simple",
args: args{"simple"},
wantErr: false,
},
{
name: "storeExists",
args: args{"simple"},
wantErr: true,
specErr: ErrStoreExists,
},
{
name: "newStore",
args: args{"notsimple"},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := db.Init(tt.args.storeName)
if (err != nil) != tt.wantErr {
t.Errorf("[FAIL] Init() error = %v, wantErr %v", err, tt.wantErr)
}
if (err != nil) != tt.wantErr && tt.specErr != nil && !errors.Is(err, tt.specErr) {
t.Errorf("[FAIL] wanted error %e, got error %e", tt.specErr, err)
}
})
}
t.Run("withStoreTest", func(t *testing.T) {
key := []byte{51, 50}
value := []byte("string")
err := db.With("simple").Put(key, value)
t.Logf("Put Value %v at Key %v", string(value), key)
if err != nil {
t.Fatalf("[FAIL] %e", err)
}
gvalue, gerr := db.With("simple").Get(key)
if gerr != nil {
t.Fatalf("[FAIL] %e", gerr)
}
if !bytes.Equal(gvalue, value) {
t.Errorf("[FAIL] wanted %v, got %v", string(value), string(gvalue))
}
t.Logf("Got Value %v at Key %v", string(gvalue), key)
})
t.Run("withNewStoreDoesExist", func(t *testing.T) {
nope := db.WithNew("bing")
if err := nope.Put([]byte("key"), []byte("value")); err != nil {
t.Fatalf("[FAIL] %e", err)
}
err := nope.Put([]byte("bing"), []byte("bong"))
if err != nil {
t.Fatalf("[FAIL] %e", err)
}
yup := db.WithNew("bing")
res, err := yup.Get([]byte("bing"))
if err != nil {
t.Errorf("[FAIL] %e", err)
}
if !bytes.Equal(res, []byte("bong")) {
t.Errorf("[FAIL] wanted %v, got %v", string([]byte("bong")), string(res))
}
})
t.Run("withNewStoreDoesntExist", func(t *testing.T) {
if nope := db.WithNew("asdfqwerty"); nope.Backend() == nil {
t.Fatalf("[FAIL] got nil result for nonexistent store when it should have made itself: %T, %v", nope, nope)
} else {
t.Logf("[SUCCESS] got nil Value for store that doesn't exist")
}
})
t.Run("withStoreDoesntExist", func(t *testing.T) {
nope := db.With(c.RandStr(10))
if nope != nil {
t.Fatalf("[FAIL] got non nil result for nonexistent store: %T, %v", nope.Backend(), nope.Backend())
} else {
t.Logf("[SUCCESS] got nil Value for store that doesn't exist")
}
})
t.Run("syncAllShouldFail", func(t *testing.T) {
db.(*DB).store["wtf"] = &Store{}
t.Cleanup(func() {
t.Logf("deleting bogus store map entry")
delete(db.(*DB).store, "wtf")
})
err := db.SyncAll()
if err == nil {
t.Fatalf("[FAIL] we should have gotten an error from bogus store map entry")
}
t.Logf("[SUCCESS] got compound error: %e", err)
})
// TODO: make sure sync is ACTUALLY sycing instead of only checking for nil err... ( ._. )
t.Run("syncAll", func(t *testing.T) {
err := db.SyncAll()
if err != nil {
t.Fatalf("[FAIL] got compound error: %e", err)
}
})
t.Run("closeAll", func(t *testing.T) {
t.Cleanup(func() {
err := os.RemoveAll("./testdata")
if err != nil {
t.Fatalf("[CLEANUP FAIL] %e", err)
}
t.Logf("[CLEANUP] cleaned up ./testdata")
})
err := db.CloseAll()
if err != nil {
t.Fatalf("[FAIL] got compound error: %e", err)
}
db = nil
})
t.Run("SyncAndCloseAll", func(t *testing.T) {
db = newTestDB(t)
seedRandStores(db, t)
err := db.SyncAndCloseAll()
if err != nil {
t.Errorf("[FAIL] failed to SyncAndCloseAll: %e", err)
}
})
}
func Test_Sync(t *testing.T) {
// TODO: make sure sync is ACTUALLY sycing instead of only checking for nil err...
var db = newTestDB(t)
seedRandStores(db, t)
t.Run("Sync", func(t *testing.T) {
for d := range db.(*DB).store {
err := db.With(d).Sync()
if err != nil {
t.Errorf("[FAIL] failed to sync %s: %e", d, err)
} else {
t.Logf("[+] Sync() successful for %s", d)
}
}
})
}
func Test_Close(t *testing.T) {
var db = newTestDB(t)
defer func() {
db = nil
}()
seedRandStores(db, t)
var oldstores []string
t.Run("Close", func(t *testing.T) {
for d := range db.AllStores() {
oldstores = append(oldstores, d)
err := db.Close(d)
if err != nil {
t.Fatalf("[FAIL] failed to close %s: %v", d, err)
}
t.Logf("[+] Close() successful for %s", d)
}
t.Run("AssureClosed", func(t *testing.T) {
for _, d := range oldstores {
if st := db.With(d); st != nil {
spew.Dump(st)
t.Errorf("[FAIL] store %s should have been deleted", d)
}
}
t.Logf("[SUCCESS] Confirmed that all stores have been closed")
})
})
t.Run("CantCloseBogusStore", func(t *testing.T) {
err := db.Close(c.RandStr(55))
if !errors.Is(err, ErrBogusStore) {
t.Errorf("[FAIL] got err %e, wanted err %e", err, ErrBogusStore)
}
})
}
func Test_withAll(t *testing.T) {
var db = newTestDB(t)
asdf1 := c.RandStr(10)
asdf2 := c.RandStr(10)
defer func() {
if err := db.CloseAll(); err != nil && !errors.Is(err, fs.ErrClosed) {
t.Errorf("[FAIL] failed to close all stores: %v", err)
}
}()
t.Run("withAllNoStores", func(t *testing.T) {
err := db.(*DB).withAll(121)
if !errors.Is(err, ErrNoStores) {
t.Errorf("[FAIL] got err %e, wanted err %e", err, ErrNoStores)
}
})
t.Run("withAllNilMap", func(t *testing.T) {
nilDb := newTestDB(t)
nilDb.(*DB).store = nil
err := nilDb.(*DB).withAll(dclose)
if err == nil {
t.Errorf("[FAIL] got nil err from trying to work on nil map, wanted err")
}
})
t.Run("withAllBogusAction", func(t *testing.T) {
err := db.Init(asdf1)
if err != nil {
t.Errorf("[FAIL] unexpected error: %e", err)
}
wAllErr := db.(*DB).withAll(121)
if !errors.Is(wAllErr, ErrUnknownAction) {
t.Errorf("[FAIL] wanted error %e, got error %e", ErrUnknownAction, err)
}
})
t.Run("ListAll", func(t *testing.T) {
allStores := db.AllStores()
if len(allStores) == 0 {
t.Errorf("[FAIL] no stores found")
}
for n, s := range allStores {
if n == "" {
t.Errorf("[FAIL] store name is empty")
}
if s == nil {
t.Errorf("[FAIL] store is nil")
}
t.Logf("[+] found store named %s: %v", n, s)
}
if len(allStores) != len(db.(*DB).store) {
t.Errorf("[FAIL] found %d stores, expected %d", len(allStores), len(db.(*DB).store))
}
})
t.Run("ListAllAndInteract", func(t *testing.T) {
err := db.Init(asdf2)
if err != nil {
t.Errorf("[FAIL] unexpected error: %e", err)
}
err = db.With(asdf1).Put([]byte("asdf"), []byte("asdf"))
if err != nil {
t.Errorf("[FAIL] unexpected error: %e", err)
}
err = db.With(asdf2).Put([]byte("asdf"), []byte("asdf"))
if err != nil {
t.Errorf("[FAIL] unexpected error: %e", err)
}
allStores := db.AllStores()
if len(allStores) == 0 {
t.Errorf("[FAIL] no stores found")
}
for n, s := range allStores {
if n == "" {
t.Errorf("[FAIL] store name is empty")
}
if s == nil {
t.Errorf("[FAIL] store is nil")
}
if len(db.(*DB).store) != 2 {
t.Errorf("[SANITY FAIL] found %d stores, expected %d", len(allStores), len(db.(*DB).store))
}
t.Logf("[+] found store named %s: %v", n, s)
if len(allStores) != len(db.(*DB).store) {
t.Errorf("[FAIL] found %d stores, expected %d", len(allStores), len(db.(*DB).store))
}
var res []byte
res, err = db.With(n).Get([]byte("asdf"))
if err != nil {
t.Errorf("[FAIL] unexpected error: %v", err)
}
if !bytes.Equal(res, []byte("asdf")) {
t.Errorf("[FAIL] expected %s, got %s", n, res)
} else {
t.Logf("[+] found %s in store %s", res, n)
}
}
})
t.Run("WithAllIncludingBadStore", func(t *testing.T) {
db.(*DB).store["yeeterson"] = &Store{}
err := db.(*DB).withAll(dclose)
if err == nil {
t.Errorf("[FAIL] got nil err, wanted any error")
}
delete(db.(*DB).store, "yeeterson")
})
}
func Test_WithOptions(t *testing.T) { //nolint:funlen,gocognit,cyclop
tpath := t.TempDir()
tdb := OpenDB(tpath)
if tdb == nil {
t.Fatalf("failed to open testdb at %s, got nil", tpath)
}
// FIXME: inconsistent with other implementations (bitcask)
defer func() {
t.Helper()
err := tdb.CloseAll()
if err == nil {
t.Fatalf("[FAIL] was able to close uninitialized store, expected error")
}
}()
t.Run("InitWithBogusOption", func(t *testing.T) {
db := newTestDB(t)
err := db.Init("bogus", "yeet")
if err == nil {
t.Errorf("[FAIL] Init should have failed with bogus option")
}
})
}
func Test_PhonyInit(t *testing.T) {
newtmp := t.TempDir()
err := os.MkdirAll(newtmp+"/"+t.Name(), 0755)
if err != nil {
t.Fatalf("[FAIL] failed to create test directory: %e", err)
}
err = os.Symlink("/dev/null", filepath.Join(newtmp, t.Name(), "lock"))
if err != nil {
t.Fatal(err.Error())
}
tdb := OpenDB(newtmp)
defer func() {
_ = tdb.CloseAll()
}()
err = tdb.Init(t.Name())
if err == nil {
t.Error("[FAIL] expected error while trying to open a store where lock exists, got nil")
}
}