Performance and testing improvements

Signed-off-by: kayos@tcp.direct <kayos@tcp.direct>

go.mod

@@ -1,12 +1,12 @@
 module git.tcp.direct/kayos/common
 
-go 1.18
+go 1.19
 
 require (
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.27.0
-	golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9
+	golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8
-	inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
+	inet.af/netaddr v0.0.0-20220811202034-502d2d690317
 	nullprogram.com/x/rng v1.1.0
 )
 
@@ -14,6 +14,6 @@ require (
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
-	go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
 	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
 )

go.sum

@@ -13,13 +13,14 @@ github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Q
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE=
 go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA=
-go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 h1:Tx9kY6yUkLge/pFG7IEMwDZy6CS2ajFc9TvQdPCW0uA=
 go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4=
+go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c=
+golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c=
-golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -41,7 +42,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw=
+inet.af/netaddr v0.0.0-20220811202034-502d2d690317 h1:U2fwK6P2EqmopP/hFLTOAjWTki0qgd4GMJn5X8wOleU=
-inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8=
+inet.af/netaddr v0.0.0-20220811202034-502d2d690317/go.mod h1:OIezDfdzOgFhuw4HuWapWq2e9l0H9tK4F1j+ETRtF3k=
 nullprogram.com/x/rng v1.1.0 h1:SMU7DHaQSWtKJNTpNFIFt8Wd/KSmOuSDPXrMFp/UMro=
 nullprogram.com/x/rng v1.1.0/go.mod h1:glGw6V87vyfawxCzqOABL3WfL95G65az9Z2JZCylCkg=

hash/hash.go

@@ -2,13 +2,14 @@ package hash
 
 import (
 	"bytes"
-	"crypto/md5"
+	"crypto/md5"  //nolint:gosec
-	"crypto/sha1"
+	"crypto/sha1" //nolint:gosec
 	"crypto/sha256"
 	"crypto/sha512"
 	"hash"
 	"io"
 	"os"
+	"sync"
 
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/blake2b"
@@ -25,26 +26,60 @@ const (
 	TypeMD5
 )
 
+var (
+	sha1Pool = &sync.Pool{
+		New: func() interface{} {
+			return sha1.New() //nolint:gosec
+		},
+	}
+	sha256Pool = &sync.Pool{
+		New: func() interface{} {
+			return sha256.New()
+		},
+	}
+	sha512Pool = &sync.Pool{
+		New: func() interface{} {
+			return sha512.New()
+		},
+	}
+	md5Pool = &sync.Pool{
+		New: func() interface{} {
+			return md5.New() //nolint:gosec
+		},
+	}
+	blake2bPool = &sync.Pool{
+		New: func() interface{} {
+			h, _ := blake2b.New(blake2b.Size, nil)
+			return h
+		},
+	}
+)
+
 func Sum(ht Type, b []byte) []byte {
 	var h hash.Hash
 	switch ht {
 	case TypeBlake2b:
-		h, _ := blake2b.New(blake2b.Size, nil)
+		h = blake2bPool.Get().(hash.Hash)
-		h.Write(b)
+		defer blake2bPool.Put(h)
-		return h.Sum(nil)
 	case TypeSHA1:
-		h = sha1.New()
+		h = sha1Pool.Get().(hash.Hash)
+		defer sha1Pool.Put(h)
 	case TypeSHA256:
-		h = sha256.New()
+		h = sha256Pool.Get().(hash.Hash)
+		defer sha256Pool.Put(h)
 	case TypeSHA512:
-		h = sha512.New()
+		h = sha512Pool.Get().(hash.Hash)
+		defer sha512Pool.Put(h)
 	case TypeMD5:
-		h = md5.New()
+		h = md5Pool.Get().(hash.Hash)
+		defer md5Pool.Put(h)
 	default:
 		return nil
 	}
 	h.Write(b)
-	return h.Sum(nil)
+	sum := h.Sum(nil)
+	h.Reset()
+	return sum
 }
 
 // Blake2bSum ignores all errors and gives you a blakae2b 64 hash value as a byte slice. (or panics somehow)

hash/hash_test.go

@@ -2,6 +2,7 @@ package hash
 
 import (
 	"bytes"
+	"encoding/base64"
 	"os"
 	"testing"
 
@@ -71,33 +72,33 @@ func TestSum(t *testing.T) {
 	}
 
 	var (
-		ogsha1    = squish.B64d(kayosSHA1)
+		ogsha1, _   = base64.StdEncoding.DecodeString(kayosSHA1)
-		ogsha256  = squish.B64d(kayosSHA256)
+		ogsha256, _ = base64.StdEncoding.DecodeString(kayosSHA256)
-		ogsha512  = squish.B64d(kayosSHA512)
+		ogsha512, _ = base64.StdEncoding.DecodeString(kayosSHA512)
-		ogmd5     = squish.B64d(kayosMD5)
+		ogmd5, _    = base64.StdEncoding.DecodeString(kayosMD5)
-		newsha1   = Sum(TypeSHA1, []byte("kayos\n"))
+		newsha1     = Sum(TypeSHA1, []byte("kayos\n"))
-		newsha256 = Sum(TypeSHA256, []byte("kayos\n"))
+		newsha256   = Sum(TypeSHA256, []byte("kayos\n"))
-		newsha512 = Sum(TypeSHA512, []byte("kayos\n"))
+		newsha512   = Sum(TypeSHA512, []byte("kayos\n"))
-		newmd5    = Sum(TypeMD5, []byte("kayos\n"))
+		newmd5      = Sum(TypeMD5, []byte("kayos\n"))
 	)
 
 	if !bytes.Equal(newsha1, ogsha1) {
-		t.Fatalf("[sha1] wanted: %v, got %v", kayosSHA1, squish.B64e(newsha1))
+		t.Fatalf("[sha1] wanted: %v, got %v", ogsha1, newsha1)
 	}
 	t.Logf("[sha1]   success: %s", kayosSHA1)
 
 	if !bytes.Equal(newsha256, ogsha256) {
-		t.Fatalf("[sha256] wanted: %v, got %v", kayosSHA256, squish.B64e(newsha256))
+		t.Fatalf("[sha256] wanted: %v, got %v", ogsha256, newsha256)
 	}
 	t.Logf("[sha256] success: %s", kayosSHA256)
 
 	if !bytes.Equal(newsha512, ogsha512) {
-		t.Fatalf("[sha512] wanted: %v, got %v", kayosSHA512, squish.B64e(newsha512))
+		t.Fatalf("[sha512] wanted: %v, got %v", ogsha512, newsha512)
 	}
 	t.Logf("[sha512] success: %s", kayosSHA512)
 
 	if !bytes.Equal(newmd5, ogmd5) {
-		t.Fatalf("[md5] wanted: %v, got %v", kayosMD5, squish.B64e(newmd5))
+		t.Fatalf("[md5] wanted: %v, got %v", ogmd5, newmd5)
 	}
 	t.Logf("[md5]    success: %s", kayosMD5)
 }

squish/squish.go

@@ -4,46 +4,94 @@ import (
 	"bytes"
 	"compress/gzip"
 	"encoding/base64"
+	"errors"
 	"io"
+	"sync"
+)
+
+var (
+	bufPool = &sync.Pool{
+		New: func() interface{} {
+			return &bytes.Buffer{}
+		},
+	}
+
+	gzipPool = &sync.Pool{
+		New: func() interface{} {
+			return gzip.NewWriter(nil)
+		},
+	}
 )
 
 // Gzip compresses as slice of bytes using gzip compression.
 func Gzip(data []byte) []byte {
-	var b bytes.Buffer
+	buf := bufPool.Get().(*bytes.Buffer)
-	gz := gzip.NewWriter(&b)
+	gz := gzipPool.Get().(*gzip.Writer)
-	// In theory this should never fail, and I don't know how to make the gzip buffered reader fail in testing.
+	buf.Reset()
-	_, _ = gz.Write(data)
+	r, w := io.Pipe()
-	_ = gz.Close()
+	gz.Reset(w)
-	return b.Bytes()
+	go func() {
+		_, _ = gz.Write(data)
+		_ = gz.Close()
+		_ = w.Close()
+	}()
+	n, _ := buf.ReadFrom(r)
+	buf.Truncate(int(n))
+	_ = r.Close()
+	res, _ := io.ReadAll(buf)
+	bufPool.Put(buf)
+	gzipPool.Put(gz)
+	return res
 }
 
 // Gunzip decompresses a gzip compressed slice of bytes.
 func Gunzip(data []byte) (out []byte, err error) {
-	var gz *gzip.Reader
+	gz, err := gzip.NewReader(bytes.NewReader(data))
-	gz, err = gzip.NewReader(bytes.NewReader(data))
 	if err != nil {
-		return
+		return nil, err
 	}
-	return io.ReadAll(gz)
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	var n int64
+	n, _ = buf.ReadFrom(gz)
+	err = gz.Close()
+	buf.Truncate(int(n))
+	res, _ := io.ReadAll(buf)
+	bufPool.Put(buf)
+	return res, err
 }
 
 // B64e encodes the given slice of bytes into base64 standard encoding.
-func B64e(cytes []byte) (data string) {
+func B64e(in []byte) (out string) {
-	data = base64.StdEncoding.EncodeToString(cytes)
+	buf := bufPool.Get().(*bytes.Buffer)
-	return
+	buf.Reset()
+	buf.Grow(base64.StdEncoding.EncodedLen(len(in)))
+	b64 := base64.NewEncoder(base64.StdEncoding, buf)
+	_, _ = b64.Write(in)
+	_ = b64.Close()
+	res := buf.Bytes()
+	bufPool.Put(buf)
+	return string(res)
 }
 
 // B64d decodes the given string into the original slice of bytes.
 // Do note that this is for non critical tasks, it has no error handling for purposes of clean code.
 func B64d(str string) (data []byte) {
+	if len(str) == 0 {
+		return nil
+	}
 	data, _ = base64.StdEncoding.DecodeString(str)
 	return data
 }
 
 // UnpackStr UNsafely unpacks (usually banners) that have been base64'd and then gzip'd.
 func UnpackStr(encoded string) (string, error) {
-	dcytes, err := Gunzip(B64d(encoded))
+	one := B64d(encoded)
-	if err != nil {
+	if len(one) == 0 {
+		return "", errors.New("0 length base64 decoding result")
+	}
+	dcytes, err := Gunzip(one)
+	if err != nil && !errors.Is(err, io.EOF) {
 		return "", err
 	}
 	return string(dcytes), nil

squish/squish_test.go

@@ -2,7 +2,10 @@ package squish
 
 import (
 	"bytes"
+	"encoding/base64"
 	"testing"
+
+	"git.tcp.direct/kayos/common/entropy"
 )
 
 const lip string = `
@@ -32,7 +35,7 @@ func TestGzip(t *testing.T) {
 	t.Logf("[PASS] Gzip compress succeeded, squished %d bytes.", profit)
 	hosDown, err := Gunzip(gsUp)
 	if err != nil {
-		t.Fatalf("Gzip decompression failed: %e", err)
+		t.Fatalf("Gzip decompression failed: %s", err.Error())
 	}
 	if !bytes.Equal(hosDown, []byte(lip)) {
 		t.Fatalf("[FAIL] Gzip decompression failed, data does not appear to be the same after decompression")
@@ -46,14 +49,114 @@ func TestGzip(t *testing.T) {
 		t.Fatalf("[FAIL] Gunzip didn't fail on nil input")
 	}
 }
-func TestUnpackStr(t *testing.T) {
+
-	packed := B64e(Gzip([]byte(lip)))
+func TestGunzipMustFails(t *testing.T) {
+	blank := ""
+	_, err := Gunzip([]byte(blank))
+	if err == nil {
+		t.Fatalf("[FAIL] Gunzip didn't fail on empty input")
+	}
+	_, err = UnpackStr(blank)
+	if err == nil {
+		t.Fatalf("[FAIL] UnpackStr didn't fail on empty input")
+	}
+	junk := "junk"
+	_, err = Gunzip([]byte(junk))
+	if err == nil {
+		t.Fatalf("[FAIL] Gunzip didn't fail on junk input")
+	}
+	_, err = UnpackStr(junk)
+	if err == nil {
+		t.Fatalf("[FAIL] UnpackStr didn't fail on junk input")
+	}
+}
+
+func TestGzipEntropic(t *testing.T) {
+	for i := 0; i < 50; i++ {
+		dat := []byte(entropy.RandStr(entropy.RNG(55) * 1024))
+		for len(dat) < 1024 {
+			dat = []byte(entropy.RandStr(entropy.RNG(55) * 1024))
+		}
+		gzTest(dat, t)
+	}
+}
+
+func gzTest(dat []byte, t *testing.T) {
+	t.Logf("Testing Gzip on %d bytes of data", len(dat))
+	gsUp := Gzip(dat)
+	if bytes.Equal(gsUp, dat) {
+		t.Fatalf("[FAIL] Gzip didn't change the data at all despite being error free...")
+	}
+	if len(gsUp) == len(dat) || len(gsUp) > len(dat) {
+		t.Fatalf("[FAIL] Gzip didn't change the sise of the data at all (or it grew)... before: %d  after: %d",
+			len(dat), len(gsUp))
+	}
+	if len(gsUp) == 0 {
+		t.Fatalf("[FAIL] ended up with 0 bytes after compression...")
+	}
+	profit := len(dat) - len(gsUp)
+	t.Logf("[PASS] Gzip compress succeeded, squished %d bytes.", profit)
+	hosDown, err := Gunzip(gsUp)
+	if err != nil {
+		t.Fatalf("Gzip decompression failed: %s", err.Error())
+	}
+	if !bytes.Equal(hosDown, dat) {
+		t.Fatalf("[FAIL] Gzip decompression failed, data does not appear to be the same after decompression")
+	}
+	if len(hosDown) != len(dat) {
+		t.Fatalf("[FAIL] Gzip decompression failed, data [%d] does not appear to be the same [%d] length after decompression", hosDown, len(dat))
+	}
+	t.Logf("[PASS] Gzip decompress succeeded, restored %d bytes.", profit)
+}
+
+func TestGzipDeterministic(t *testing.T) {
+	packed := Gzip([]byte(lip))
+	for n := 0; n < 10; n++ {
+		again := Gzip([]byte(lip))
+		if !bytes.Equal(again, packed) {
+			t.Fatalf("[FAIL] Gzip is not deterministic")
+		}
+	}
+}
+
+func TestUnpackStr(t *testing.T) { //nolint:cyclop
+	gzd := Gzip([]byte(lip))
+	if len(gzd) == 0 {
+		t.Fatalf("[FAIL] Gzip failed to compress data")
+	}
+	gzdSanity, gzdErr := Gunzip(gzd)
+	if gzdErr != nil {
+		t.Fatalf("Gzip failed: %s", gzdErr.Error())
+	}
+	if !bytes.Equal(gzdSanity, []byte(lip)) {
+		t.Fatalf("Bytes not equal after Gzip: %v != %v", gzdSanity, []byte(lip))
+	}
+	packed := B64e(gzd)
+	if len(packed) == 0 {
+		t.Fatalf("[FAIL] B64e failed to encode data")
+	}
+	t.Logf("Packed: %s", packed)
+	sanity1, err1 := base64.StdEncoding.DecodeString(packed)
+	if err1 != nil {
+		t.Fatalf("b64 failed: %s", err1.Error())
+	}
+	if !bytes.Equal(sanity1, gzd) {
+		t.Fatalf("Bytes not equal after b64: %v != %v", sanity1, gzd)
+	}
+	sanity2, err2 := Gunzip(sanity1)
+	if err2 != nil {
+		t.Fatalf("Gzip failed: %s", err2.Error())
+	}
+	if !bytes.Equal(sanity2, []byte(lip)) {
+		t.Fatalf("Bytes not equal after Gzip: %v != %v", sanity2, []byte(lip))
+	}
 	unpacked, err := UnpackStr(packed)
 	switch {
 	case err != nil:
-		t.Fatalf("[FAIL] %e", err)
+		t.Errorf("[FAIL] %s", err.Error())
 	case unpacked != lip:
-		t.Fatalf("[FAIL] unpackstr decided to not work, who knows why. If you see this than I have already become a janitor.")
+		t.Fatalf("[FAIL] unpackstr decided to not work, who knows why. If you see this than I have already become a janitor.\n"+
+			"unpacked: %s != packed: %s", unpacked, lip)
 	default:
 		t.Logf("[PASS] TestUnpackStr")
 	}