From 374ca0d5fb742444146181542e3ebdc67ef95311 Mon Sep 17 00:00:00 2001 From: "kayos@tcp.direct" Date: Sun, 28 Jan 2024 21:10:36 -0800 Subject: [PATCH] Feat: Discover automatically recovers from bad meta.json --- bitcask/bitcask.go | 36 ++++++++++++++++++++++++++++++++++++ bitcask/bitcask_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/bitcask/bitcask.go b/bitcask/bitcask.go index cf34a2e..e494e7c 100644 --- a/bitcask/bitcask.go +++ b/bitcask/bitcask.go @@ -74,8 +74,40 @@ func (db *DB) Discover() ([]string, error) { if _, ok := db.store[name]; ok { continue } + recoverOnce := &sync.Once{} + openUp: c, e := bitcask.Open(filepath.Join(db.path, name), defaultBitcaskOptions...) if e != nil { + retry := false + recoverOnce.Do(func() { + metaErr := new(bitcask.ErrBadMetadata) + if !errors.As(e, &metaErr) { + return + } + if !strings.Contains(metaErr.Error(), "unexpected end of JSON input") { + return + } + if c != nil { + _ = c.Close() + } + println("WARN: bitcask store", name, "has bad metadata, attempting to repair") + oldMeta := filepath.Join(db.path, name, "meta.json") + newMeta := filepath.Join(db.path, name, "meta.json.backup") + println("WARN: renaming", oldMeta, "to", newMeta) + // likely defunct lockfile is present too, remove it + if osErr := os.Rename(oldMeta, newMeta); osErr != nil { + println("WARN: failed to rename", oldMeta, "to", newMeta, ":", osErr) + return + } + if _, serr := os.Stat(filepath.Join(db.path, name, "lock")); serr == nil { + println("WARN: removing defunct lockfile") + _ = os.Remove(filepath.Join(db.path, name, "lock")) + } + retry = true + }) + if retry { + goto openUp + } errs = append(errs, e) continue } @@ -84,6 +116,10 @@ func (db *DB) Discover() ([]string, error) { } for _, e := range errs { + if err == nil { + err = e + continue + } err = fmt.Errorf("%w: %v", err, e) } diff --git a/bitcask/bitcask_test.go b/bitcask/bitcask_test.go index 820bf3f..3876ca0 100644 --- a/bitcask/bitcask_test.go +++ b/bitcask/bitcask_test.go @@ -5,6 +5,7 @@ import ( "errors" "io/fs" "os" + "path/filepath" "strings" "testing" @@ -198,6 +199,29 @@ func TestDB_Init(t *testing.T) { //nolint:funlen,gocognit,cyclop } } }) + t.Run("RecoverBadMetaJSON", func(t *testing.T) { + var path string + path, db = newTestDB(t) + names := seedRandStores(db, t) + err := db.SyncAndCloseAll() + if err != nil { + t.Errorf("[FAIL] failed to SyncAndCloseAll: %e", err) + } + if err = os.WriteFile(filepath.Join(path, names[0], "meta.json"), []byte(""), 0644); err != nil { + t.Fatalf("[FAIL] failed to write bad meta.json: %e", err) + } + db = OpenDB(path) + _, err = db.(*DB).Discover() + if err != nil { + t.Errorf("[FAIL] failed to discover stores: %e", err) + } + if err = db.With(names[0]).Put([]byte("asdf"), []byte("asdf")); err != nil { + t.Errorf("[FAIL] failed to put value: %e", err) + } + if err = db.SyncAndCloseAll(); err != nil { + t.Errorf("[FAIL] failed to SyncAndCloseAll: %e", err) + } + }) } func Test_Sync(t *testing.T) {