Feat: Discover automatically recovers from bad meta.json

bitcask/bitcask.go

@@ -74,8 +74,40 @@ func (db *DB) Discover() ([]string, error) {
 		if _, ok := db.store[name]; ok {
 			continue
 		}
+		recoverOnce := &sync.Once{}
+	openUp:
 		c, e := bitcask.Open(filepath.Join(db.path, name), defaultBitcaskOptions...)
 		if e != nil {
+			retry := false
+			recoverOnce.Do(func() {
+				metaErr := new(bitcask.ErrBadMetadata)
+				if !errors.As(e, &metaErr) {
+					return
+				}
+				if !strings.Contains(metaErr.Error(), "unexpected end of JSON input") {
+					return
+				}
+				if c != nil {
+					_ = c.Close()
+				}
+				println("WARN: bitcask store", name, "has bad metadata, attempting to repair")
+				oldMeta := filepath.Join(db.path, name, "meta.json")
+				newMeta := filepath.Join(db.path, name, "meta.json.backup")
+				println("WARN: renaming", oldMeta, "to", newMeta)
+				// likely defunct lockfile is present too, remove it
+				if osErr := os.Rename(oldMeta, newMeta); osErr != nil {
+					println("WARN: failed to rename", oldMeta, "to", newMeta, ":", osErr)
+					return
+				}
+				if _, serr := os.Stat(filepath.Join(db.path, name, "lock")); serr == nil {
+					println("WARN: removing defunct lockfile")
+					_ = os.Remove(filepath.Join(db.path, name, "lock"))
+				}
+				retry = true
+			})
+			if retry {
+				goto openUp
+			}
 			errs = append(errs, e)
 			continue
 		}
@@ -84,6 +116,10 @@ func (db *DB) Discover() ([]string, error) {
 	}
 
 	for _, e := range errs {
+		if err == nil {
+			err = e
+			continue
+		}
 		err = fmt.Errorf("%w: %v", err, e)
 	}
 

bitcask/bitcask_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io/fs"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -198,6 +199,29 @@ func TestDB_Init(t *testing.T) { //nolint:funlen,gocognit,cyclop
 			}
 		}
 	})
+	t.Run("RecoverBadMetaJSON", func(t *testing.T) {
+		var path string
+		path, db = newTestDB(t)
+		names := seedRandStores(db, t)
+		err := db.SyncAndCloseAll()
+		if err != nil {
+			t.Errorf("[FAIL] failed to SyncAndCloseAll: %e", err)
+		}
+		if err = os.WriteFile(filepath.Join(path, names[0], "meta.json"), []byte(""), 0644); err != nil {
+			t.Fatalf("[FAIL] failed to write bad meta.json: %e", err)
+		}
+		db = OpenDB(path)
+		_, err = db.(*DB).Discover()
+		if err != nil {
+			t.Errorf("[FAIL] failed to discover stores: %e", err)
+		}
+		if err = db.With(names[0]).Put([]byte("asdf"), []byte("asdf")); err != nil {
+			t.Errorf("[FAIL] failed to put value: %e", err)
+		}
+		if err = db.SyncAndCloseAll(); err != nil {
+			t.Errorf("[FAIL] failed to SyncAndCloseAll: %e", err)
+		}
+	})
 }
 
 func Test_Sync(t *testing.T) {