Performance and testing improvements

Signed-off-by: kayos@tcp.direct <kayos@tcp.direct>
This commit is contained in:
kayos@tcp.direct 2022-08-22 06:14:08 -07:00
parent 945ce6ee14
commit 9d344e77b5
Signed by: kayos
GPG Key ID: 4B841471B4BEE979
6 changed files with 239 additions and 51 deletions

8
go.mod
View File

@ -1,12 +1,12 @@
module git.tcp.direct/kayos/common
go 1.18
go 1.19
require (
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.27.0
golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9
inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8
inet.af/netaddr v0.0.0-20220811202034-502d2d690317
nullprogram.com/x/rng v1.1.0
)
@ -14,6 +14,6 @@ require (
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect
go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
)

11
go.sum
View File

@ -13,13 +13,14 @@ github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Q
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE=
go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA=
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 h1:Tx9kY6yUkLge/pFG7IEMwDZy6CS2ajFc9TvQdPCW0uA=
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4=
go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c=
golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c=
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -41,7 +42,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw=
inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8=
inet.af/netaddr v0.0.0-20220811202034-502d2d690317 h1:U2fwK6P2EqmopP/hFLTOAjWTki0qgd4GMJn5X8wOleU=
inet.af/netaddr v0.0.0-20220811202034-502d2d690317/go.mod h1:OIezDfdzOgFhuw4HuWapWq2e9l0H9tK4F1j+ETRtF3k=
nullprogram.com/x/rng v1.1.0 h1:SMU7DHaQSWtKJNTpNFIFt8Wd/KSmOuSDPXrMFp/UMro=
nullprogram.com/x/rng v1.1.0/go.mod h1:glGw6V87vyfawxCzqOABL3WfL95G65az9Z2JZCylCkg=

View File

@ -2,13 +2,14 @@ package hash
import (
"bytes"
"crypto/md5"
"crypto/sha1"
"crypto/md5" //nolint:gosec
"crypto/sha1" //nolint:gosec
"crypto/sha256"
"crypto/sha512"
"hash"
"io"
"os"
"sync"
"github.com/pkg/errors"
"golang.org/x/crypto/blake2b"
@ -25,26 +26,60 @@ const (
TypeMD5
)
var (
sha1Pool = &sync.Pool{
New: func() interface{} {
return sha1.New() //nolint:gosec
},
}
sha256Pool = &sync.Pool{
New: func() interface{} {
return sha256.New()
},
}
sha512Pool = &sync.Pool{
New: func() interface{} {
return sha512.New()
},
}
md5Pool = &sync.Pool{
New: func() interface{} {
return md5.New() //nolint:gosec
},
}
blake2bPool = &sync.Pool{
New: func() interface{} {
h, _ := blake2b.New(blake2b.Size, nil)
return h
},
}
)
func Sum(ht Type, b []byte) []byte {
var h hash.Hash
switch ht {
case TypeBlake2b:
h, _ := blake2b.New(blake2b.Size, nil)
h.Write(b)
return h.Sum(nil)
h = blake2bPool.Get().(hash.Hash)
defer blake2bPool.Put(h)
case TypeSHA1:
h = sha1.New()
h = sha1Pool.Get().(hash.Hash)
defer sha1Pool.Put(h)
case TypeSHA256:
h = sha256.New()
h = sha256Pool.Get().(hash.Hash)
defer sha256Pool.Put(h)
case TypeSHA512:
h = sha512.New()
h = sha512Pool.Get().(hash.Hash)
defer sha512Pool.Put(h)
case TypeMD5:
h = md5.New()
h = md5Pool.Get().(hash.Hash)
defer md5Pool.Put(h)
default:
return nil
}
h.Write(b)
return h.Sum(nil)
sum := h.Sum(nil)
h.Reset()
return sum
}
// Blake2bSum ignores all errors and gives you a blakae2b 64 hash value as a byte slice. (or panics somehow)

View File

@ -2,6 +2,7 @@ package hash
import (
"bytes"
"encoding/base64"
"os"
"testing"
@ -71,33 +72,33 @@ func TestSum(t *testing.T) {
}
var (
ogsha1 = squish.B64d(kayosSHA1)
ogsha256 = squish.B64d(kayosSHA256)
ogsha512 = squish.B64d(kayosSHA512)
ogmd5 = squish.B64d(kayosMD5)
newsha1 = Sum(TypeSHA1, []byte("kayos\n"))
newsha256 = Sum(TypeSHA256, []byte("kayos\n"))
newsha512 = Sum(TypeSHA512, []byte("kayos\n"))
newmd5 = Sum(TypeMD5, []byte("kayos\n"))
ogsha1, _ = base64.StdEncoding.DecodeString(kayosSHA1)
ogsha256, _ = base64.StdEncoding.DecodeString(kayosSHA256)
ogsha512, _ = base64.StdEncoding.DecodeString(kayosSHA512)
ogmd5, _ = base64.StdEncoding.DecodeString(kayosMD5)
newsha1 = Sum(TypeSHA1, []byte("kayos\n"))
newsha256 = Sum(TypeSHA256, []byte("kayos\n"))
newsha512 = Sum(TypeSHA512, []byte("kayos\n"))
newmd5 = Sum(TypeMD5, []byte("kayos\n"))
)
if !bytes.Equal(newsha1, ogsha1) {
t.Fatalf("[sha1] wanted: %v, got %v", kayosSHA1, squish.B64e(newsha1))
t.Fatalf("[sha1] wanted: %v, got %v", ogsha1, newsha1)
}
t.Logf("[sha1] success: %s", kayosSHA1)
if !bytes.Equal(newsha256, ogsha256) {
t.Fatalf("[sha256] wanted: %v, got %v", kayosSHA256, squish.B64e(newsha256))
t.Fatalf("[sha256] wanted: %v, got %v", ogsha256, newsha256)
}
t.Logf("[sha256] success: %s", kayosSHA256)
if !bytes.Equal(newsha512, ogsha512) {
t.Fatalf("[sha512] wanted: %v, got %v", kayosSHA512, squish.B64e(newsha512))
t.Fatalf("[sha512] wanted: %v, got %v", ogsha512, newsha512)
}
t.Logf("[sha512] success: %s", kayosSHA512)
if !bytes.Equal(newmd5, ogmd5) {
t.Fatalf("[md5] wanted: %v, got %v", kayosMD5, squish.B64e(newmd5))
t.Fatalf("[md5] wanted: %v, got %v", ogmd5, newmd5)
}
t.Logf("[md5] success: %s", kayosMD5)
}

View File

@ -4,46 +4,94 @@ import (
"bytes"
"compress/gzip"
"encoding/base64"
"errors"
"io"
"sync"
)
var (
bufPool = &sync.Pool{
New: func() interface{} {
return &bytes.Buffer{}
},
}
gzipPool = &sync.Pool{
New: func() interface{} {
return gzip.NewWriter(nil)
},
}
)
// Gzip compresses as slice of bytes using gzip compression.
func Gzip(data []byte) []byte {
var b bytes.Buffer
gz := gzip.NewWriter(&b)
// In theory this should never fail, and I don't know how to make the gzip buffered reader fail in testing.
_, _ = gz.Write(data)
_ = gz.Close()
return b.Bytes()
buf := bufPool.Get().(*bytes.Buffer)
gz := gzipPool.Get().(*gzip.Writer)
buf.Reset()
r, w := io.Pipe()
gz.Reset(w)
go func() {
_, _ = gz.Write(data)
_ = gz.Close()
_ = w.Close()
}()
n, _ := buf.ReadFrom(r)
buf.Truncate(int(n))
_ = r.Close()
res, _ := io.ReadAll(buf)
bufPool.Put(buf)
gzipPool.Put(gz)
return res
}
// Gunzip decompresses a gzip compressed slice of bytes.
func Gunzip(data []byte) (out []byte, err error) {
var gz *gzip.Reader
gz, err = gzip.NewReader(bytes.NewReader(data))
gz, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return
return nil, err
}
return io.ReadAll(gz)
buf := bufPool.Get().(*bytes.Buffer)
buf.Reset()
var n int64
n, _ = buf.ReadFrom(gz)
err = gz.Close()
buf.Truncate(int(n))
res, _ := io.ReadAll(buf)
bufPool.Put(buf)
return res, err
}
// B64e encodes the given slice of bytes into base64 standard encoding.
func B64e(cytes []byte) (data string) {
data = base64.StdEncoding.EncodeToString(cytes)
return
func B64e(in []byte) (out string) {
buf := bufPool.Get().(*bytes.Buffer)
buf.Reset()
buf.Grow(base64.StdEncoding.EncodedLen(len(in)))
b64 := base64.NewEncoder(base64.StdEncoding, buf)
_, _ = b64.Write(in)
_ = b64.Close()
res := buf.Bytes()
bufPool.Put(buf)
return string(res)
}
// B64d decodes the given string into the original slice of bytes.
// Do note that this is for non critical tasks, it has no error handling for purposes of clean code.
func B64d(str string) (data []byte) {
if len(str) == 0 {
return nil
}
data, _ = base64.StdEncoding.DecodeString(str)
return data
}
// UnpackStr UNsafely unpacks (usually banners) that have been base64'd and then gzip'd.
func UnpackStr(encoded string) (string, error) {
dcytes, err := Gunzip(B64d(encoded))
if err != nil {
one := B64d(encoded)
if len(one) == 0 {
return "", errors.New("0 length base64 decoding result")
}
dcytes, err := Gunzip(one)
if err != nil && !errors.Is(err, io.EOF) {
return "", err
}
return string(dcytes), nil

View File

@ -2,7 +2,10 @@ package squish
import (
"bytes"
"encoding/base64"
"testing"
"git.tcp.direct/kayos/common/entropy"
)
const lip string = `
@ -32,7 +35,7 @@ func TestGzip(t *testing.T) {
t.Logf("[PASS] Gzip compress succeeded, squished %d bytes.", profit)
hosDown, err := Gunzip(gsUp)
if err != nil {
t.Fatalf("Gzip decompression failed: %e", err)
t.Fatalf("Gzip decompression failed: %s", err.Error())
}
if !bytes.Equal(hosDown, []byte(lip)) {
t.Fatalf("[FAIL] Gzip decompression failed, data does not appear to be the same after decompression")
@ -46,14 +49,114 @@ func TestGzip(t *testing.T) {
t.Fatalf("[FAIL] Gunzip didn't fail on nil input")
}
}
func TestUnpackStr(t *testing.T) {
packed := B64e(Gzip([]byte(lip)))
func TestGunzipMustFails(t *testing.T) {
blank := ""
_, err := Gunzip([]byte(blank))
if err == nil {
t.Fatalf("[FAIL] Gunzip didn't fail on empty input")
}
_, err = UnpackStr(blank)
if err == nil {
t.Fatalf("[FAIL] UnpackStr didn't fail on empty input")
}
junk := "junk"
_, err = Gunzip([]byte(junk))
if err == nil {
t.Fatalf("[FAIL] Gunzip didn't fail on junk input")
}
_, err = UnpackStr(junk)
if err == nil {
t.Fatalf("[FAIL] UnpackStr didn't fail on junk input")
}
}
func TestGzipEntropic(t *testing.T) {
for i := 0; i < 50; i++ {
dat := []byte(entropy.RandStr(entropy.RNG(55) * 1024))
for len(dat) < 1024 {
dat = []byte(entropy.RandStr(entropy.RNG(55) * 1024))
}
gzTest(dat, t)
}
}
func gzTest(dat []byte, t *testing.T) {
t.Logf("Testing Gzip on %d bytes of data", len(dat))
gsUp := Gzip(dat)
if bytes.Equal(gsUp, dat) {
t.Fatalf("[FAIL] Gzip didn't change the data at all despite being error free...")
}
if len(gsUp) == len(dat) || len(gsUp) > len(dat) {
t.Fatalf("[FAIL] Gzip didn't change the sise of the data at all (or it grew)... before: %d after: %d",
len(dat), len(gsUp))
}
if len(gsUp) == 0 {
t.Fatalf("[FAIL] ended up with 0 bytes after compression...")
}
profit := len(dat) - len(gsUp)
t.Logf("[PASS] Gzip compress succeeded, squished %d bytes.", profit)
hosDown, err := Gunzip(gsUp)
if err != nil {
t.Fatalf("Gzip decompression failed: %s", err.Error())
}
if !bytes.Equal(hosDown, dat) {
t.Fatalf("[FAIL] Gzip decompression failed, data does not appear to be the same after decompression")
}
if len(hosDown) != len(dat) {
t.Fatalf("[FAIL] Gzip decompression failed, data [%d] does not appear to be the same [%d] length after decompression", hosDown, len(dat))
}
t.Logf("[PASS] Gzip decompress succeeded, restored %d bytes.", profit)
}
func TestGzipDeterministic(t *testing.T) {
packed := Gzip([]byte(lip))
for n := 0; n < 10; n++ {
again := Gzip([]byte(lip))
if !bytes.Equal(again, packed) {
t.Fatalf("[FAIL] Gzip is not deterministic")
}
}
}
func TestUnpackStr(t *testing.T) { //nolint:cyclop
gzd := Gzip([]byte(lip))
if len(gzd) == 0 {
t.Fatalf("[FAIL] Gzip failed to compress data")
}
gzdSanity, gzdErr := Gunzip(gzd)
if gzdErr != nil {
t.Fatalf("Gzip failed: %s", gzdErr.Error())
}
if !bytes.Equal(gzdSanity, []byte(lip)) {
t.Fatalf("Bytes not equal after Gzip: %v != %v", gzdSanity, []byte(lip))
}
packed := B64e(gzd)
if len(packed) == 0 {
t.Fatalf("[FAIL] B64e failed to encode data")
}
t.Logf("Packed: %s", packed)
sanity1, err1 := base64.StdEncoding.DecodeString(packed)
if err1 != nil {
t.Fatalf("b64 failed: %s", err1.Error())
}
if !bytes.Equal(sanity1, gzd) {
t.Fatalf("Bytes not equal after b64: %v != %v", sanity1, gzd)
}
sanity2, err2 := Gunzip(sanity1)
if err2 != nil {
t.Fatalf("Gzip failed: %s", err2.Error())
}
if !bytes.Equal(sanity2, []byte(lip)) {
t.Fatalf("Bytes not equal after Gzip: %v != %v", sanity2, []byte(lip))
}
unpacked, err := UnpackStr(packed)
switch {
case err != nil:
t.Fatalf("[FAIL] %e", err)
t.Errorf("[FAIL] %s", err.Error())
case unpacked != lip:
t.Fatalf("[FAIL] unpackstr decided to not work, who knows why. If you see this than I have already become a janitor.")
t.Fatalf("[FAIL] unpackstr decided to not work, who knows why. If you see this than I have already become a janitor.\n"+
"unpacked: %s != packed: %s", unpacked, lip)
default:
t.Logf("[PASS] TestUnpackStr")
}