From 7204a33512d91010835e2ad063dafe05e68f1b37 Mon Sep 17 00:00:00 2001 From: James Mills <1290234+prologic@users.noreply.github.com> Date: Thu, 8 Aug 2019 22:28:25 +1000 Subject: [PATCH] Fix and cleanup some unnecessary internal sub-packages and duplication --- bitcask.go | 3 +- internal/codec.go | 8 +-- internal/codec/codec.go | 113 ---------------------------------- internal/datafile.go | 21 +++---- internal/{model => }/entry.go | 14 ++--- 5 files changed, 20 insertions(+), 139 deletions(-) delete mode 100644 internal/codec/codec.go rename internal/{model => }/entry.go (73%) diff --git a/bitcask.go b/bitcask.go index 3707ed6..88d3ecc 100644 --- a/bitcask.go +++ b/bitcask.go @@ -15,7 +15,6 @@ import ( "github.com/gofrs/flock" "github.com/prologic/bitcask/internal" - "github.com/prologic/bitcask/internal/model" ) var ( @@ -239,7 +238,7 @@ func (b *Bitcask) put(key, value []byte) (int64, int64, error) { b.curr = curr } - e := model.NewEntry(key, value) + e := internal.NewEntry(key, value) return b.curr.Write(e) } diff --git a/internal/codec.go b/internal/codec.go index 8b46797..e43ac14 100644 --- a/internal/codec.go +++ b/internal/codec.go @@ -6,8 +6,6 @@ import ( "io" "github.com/pkg/errors" - - "github.com/prologic/bitcask/internal/model" ) const ( @@ -29,7 +27,7 @@ type Encoder struct { // Encode takes any Entry and streams it to the underlying writer. // Messages are framed with a key-length and value-length prefix. -func (e *Encoder) Encode(msg model.Entry) (int64, error) { +func (e *Encoder) Encode(msg Entry) (int64, error) { var bufKeyValue = make([]byte, ValueSize) bufKeySize := bufKeyValue[:KeySize] @@ -75,7 +73,7 @@ type Decoder struct { r io.Reader } -func (d *Decoder) Decode(v *model.Entry) (int64, error) { +func (d *Decoder) Decode(v *Entry) (int64, error) { prefixBuf := make([]byte, KeySize+ValueSize) _, err := io.ReadFull(d.r, prefixBuf) @@ -100,7 +98,7 @@ func GetKeyValueSizes(buf []byte) (uint64, uint64) { return uint64(actualKeySize), actualValueSize } -func DecodeWithoutPrefix(buf []byte, valueOffset uint64, v *model.Entry) { +func DecodeWithoutPrefix(buf []byte, valueOffset uint64, v *Entry) { v.Key = buf[:valueOffset] v.Value = buf[valueOffset : len(buf)-checksumSize] v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:]) diff --git a/internal/codec/codec.go b/internal/codec/codec.go deleted file mode 100644 index 0aa4ed3..0000000 --- a/internal/codec/codec.go +++ /dev/null @@ -1,113 +0,0 @@ -package codec - -import ( - "bufio" - "encoding/binary" - "io" - - "github.com/pkg/errors" - "github.com/prologic/bitcask/internal/model" -) - -const ( - KeySize = 4 - ValueSize = 8 - checksumSize = 4 -) - -// NewEncoder creates a streaming Entry encoder. -func NewEncoder(w io.Writer) *Encoder { - return &Encoder{w: bufio.NewWriter(w)} -} - -// Encoder wraps an underlying io.Writer and allows you to stream -// Entry encodings on it. -type Encoder struct { - w *bufio.Writer -} - -// Encode takes any Entry and streams it to the underlying writer. -// Messages are framed with a key-length and value-length prefix. -func (e *Encoder) Encode(msg model.Entry) (int64, error) { - var bufKeyValue = make([]byte, ValueSize) - - bufKeySize := bufKeyValue[:KeySize] - binary.BigEndian.PutUint32(bufKeySize, uint32(len(msg.Key))) - if _, err := e.w.Write(bufKeySize); err != nil { - return 0, errors.Wrap(err, "failed writing key length prefix") - } - - bufValueSize := bufKeyValue[:ValueSize] - binary.BigEndian.PutUint64(bufValueSize, uint64(len(msg.Value))) - if _, err := e.w.Write(bufValueSize); err != nil { - return 0, errors.Wrap(err, "failed writing value length prefix") - } - - if _, err := e.w.Write([]byte(msg.Key)); err != nil { - return 0, errors.Wrap(err, "failed writing key data") - } - if _, err := e.w.Write(msg.Value); err != nil { - return 0, errors.Wrap(err, "failed writing value data") - } - - bufChecksumSize := make([]byte, checksumSize) - binary.BigEndian.PutUint32(bufChecksumSize, msg.Checksum) - if _, err := e.w.Write(bufChecksumSize); err != nil { - return 0, errors.Wrap(err, "failed writing checksum data") - } - - if err := e.w.Flush(); err != nil { - return 0, errors.Wrap(err, "failed flushing data") - } - - return int64(KeySize + ValueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil -} - -// NewDecoder creates a streaming Entry decoder. -func NewDecoder(r io.Reader) *Decoder { - return &Decoder{r: r} -} - -// Decoder wraps an underlying io.Reader and allows you to stream -// Entry decodings on it. -type Decoder struct { - r io.Reader -} - -func (d *Decoder) Decode(v *model.Entry) (int64, error) { - prefixBuf := make([]byte, KeySize+ValueSize) - - _, err := io.ReadFull(d.r, prefixBuf) - if err != nil { - return 0, err - } - - actualKeySize, actualValueSize := GetKeyValueSizes(prefixBuf) - buf := make([]byte, actualKeySize+actualValueSize+checksumSize) - if _, err = io.ReadFull(d.r, buf); err != nil { - return 0, errors.Wrap(translateError(err), "failed reading saved data") - } - - DecodeWithoutPrefix(buf, actualValueSize, v) - return int64(KeySize + ValueSize + actualKeySize + actualValueSize + checksumSize), nil -} - -func GetKeyValueSizes(buf []byte) (uint64, uint64) { - actualKeySize := binary.BigEndian.Uint32(buf[:KeySize]) - actualValueSize := binary.BigEndian.Uint64(buf[KeySize:]) - - return uint64(actualKeySize), actualValueSize -} - -func DecodeWithoutPrefix(buf []byte, valueOffset uint64, v *model.Entry) { - v.Key = buf[:valueOffset] - v.Value = buf[valueOffset : len(buf)-checksumSize] - v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:]) -} - -func translateError(err error) error { - if err == io.EOF { - return io.ErrUnexpectedEOF - } - return err -} diff --git a/internal/datafile.go b/internal/datafile.go index 58ec8e3..b7df731 100644 --- a/internal/datafile.go +++ b/internal/datafile.go @@ -8,9 +8,6 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/mmap" - - "github.com/prologic/bitcask/internal/codec" - "github.com/prologic/bitcask/internal/model" ) const ( @@ -32,8 +29,8 @@ type Datafile struct { ra *mmap.ReaderAt w *os.File offset int64 - dec *codec.Decoder - enc *codec.Encoder + dec *Decoder + enc *Encoder } func NewDatafile(path string, id int, readonly bool) (*Datafile, error) { @@ -69,8 +66,8 @@ func NewDatafile(path string, id int, readonly bool) (*Datafile, error) { offset := stat.Size() - dec := codec.NewDecoder(r) - enc := codec.NewEncoder(w) + dec := NewDecoder(r) + enc := NewEncoder(w) return &Datafile{ id: id, @@ -122,7 +119,7 @@ func (df *Datafile) Size() int64 { return df.offset } -func (df *Datafile) Read() (e model.Entry, n int64, err error) { +func (df *Datafile) Read() (e Entry, n int64, err error) { df.Lock() defer df.Unlock() @@ -134,7 +131,7 @@ func (df *Datafile) Read() (e model.Entry, n int64, err error) { return } -func (df *Datafile) ReadAt(index, size int64) (e model.Entry, err error) { +func (df *Datafile) ReadAt(index, size int64) (e Entry, err error) { var n int b := make([]byte, size) @@ -152,13 +149,13 @@ func (df *Datafile) ReadAt(index, size int64) (e model.Entry, err error) { return } - valueOffset, _ := codec.GetKeyValueSizes(b) - codec.DecodeWithoutPrefix(b[codec.KeySize+codec.ValueSize:], valueOffset, &e) + valueOffset, _ := GetKeyValueSizes(b) + DecodeWithoutPrefix(b[KeySize+ValueSize:], valueOffset, &e) return } -func (df *Datafile) Write(e model.Entry) (int64, int64, error) { +func (df *Datafile) Write(e Entry) (int64, int64, error) { if df.w == nil { return -1, 0, ErrReadonly } diff --git a/internal/model/entry.go b/internal/entry.go similarity index 73% rename from internal/model/entry.go rename to internal/entry.go index 860ea15..4f776dd 100644 --- a/internal/model/entry.go +++ b/internal/entry.go @@ -1,16 +1,16 @@ -package model +package internal import ( "hash/crc32" ) + // Entry represents a key/value in the database type Entry struct { - Checksum uint32 - Key []byte - Offset int64 - Value []byte - } - + Checksum uint32 + Key []byte + Offset int64 + Value []byte +} func NewEntry(key, value []byte) Entry { checksum := crc32.ChecksumIEEE(value)