diff --git a/go.mod b/go.mod index 71a91b0..97f92bf 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module git.tcp.direct/kayos/common -go 1.18 +go 1.19 require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.27.0 - golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 - inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 + golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 + inet.af/netaddr v0.0.0-20220811202034-502d2d690317 nullprogram.com/x/rng v1.1.0 ) @@ -14,6 +14,6 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect - go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect ) diff --git a/go.sum b/go.sum index 6a78ecc..42ae873 100644 --- a/go.sum +++ b/go.sum @@ -13,13 +13,14 @@ github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Q github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 h1:Tx9kY6yUkLge/pFG7IEMwDZy6CS2ajFc9TvQdPCW0uA= go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c= -golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c= +golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -41,7 +42,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw= -inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8= +inet.af/netaddr v0.0.0-20220811202034-502d2d690317 h1:U2fwK6P2EqmopP/hFLTOAjWTki0qgd4GMJn5X8wOleU= +inet.af/netaddr v0.0.0-20220811202034-502d2d690317/go.mod h1:OIezDfdzOgFhuw4HuWapWq2e9l0H9tK4F1j+ETRtF3k= nullprogram.com/x/rng v1.1.0 h1:SMU7DHaQSWtKJNTpNFIFt8Wd/KSmOuSDPXrMFp/UMro= nullprogram.com/x/rng v1.1.0/go.mod h1:glGw6V87vyfawxCzqOABL3WfL95G65az9Z2JZCylCkg= diff --git a/hash/hash.go b/hash/hash.go index afd92f1..ac7c446 100644 --- a/hash/hash.go +++ b/hash/hash.go @@ -2,13 +2,14 @@ package hash import ( "bytes" - "crypto/md5" - "crypto/sha1" + "crypto/md5" //nolint:gosec + "crypto/sha1" //nolint:gosec "crypto/sha256" "crypto/sha512" "hash" "io" "os" + "sync" "github.com/pkg/errors" "golang.org/x/crypto/blake2b" @@ -25,26 +26,60 @@ const ( TypeMD5 ) +var ( + sha1Pool = &sync.Pool{ + New: func() interface{} { + return sha1.New() //nolint:gosec + }, + } + sha256Pool = &sync.Pool{ + New: func() interface{} { + return sha256.New() + }, + } + sha512Pool = &sync.Pool{ + New: func() interface{} { + return sha512.New() + }, + } + md5Pool = &sync.Pool{ + New: func() interface{} { + return md5.New() //nolint:gosec + }, + } + blake2bPool = &sync.Pool{ + New: func() interface{} { + h, _ := blake2b.New(blake2b.Size, nil) + return h + }, + } +) + func Sum(ht Type, b []byte) []byte { var h hash.Hash switch ht { case TypeBlake2b: - h, _ := blake2b.New(blake2b.Size, nil) - h.Write(b) - return h.Sum(nil) + h = blake2bPool.Get().(hash.Hash) + defer blake2bPool.Put(h) case TypeSHA1: - h = sha1.New() + h = sha1Pool.Get().(hash.Hash) + defer sha1Pool.Put(h) case TypeSHA256: - h = sha256.New() + h = sha256Pool.Get().(hash.Hash) + defer sha256Pool.Put(h) case TypeSHA512: - h = sha512.New() + h = sha512Pool.Get().(hash.Hash) + defer sha512Pool.Put(h) case TypeMD5: - h = md5.New() + h = md5Pool.Get().(hash.Hash) + defer md5Pool.Put(h) default: return nil } h.Write(b) - return h.Sum(nil) + sum := h.Sum(nil) + h.Reset() + return sum } // Blake2bSum ignores all errors and gives you a blakae2b 64 hash value as a byte slice. (or panics somehow) diff --git a/hash/hash_test.go b/hash/hash_test.go index b0cb660..78dc700 100644 --- a/hash/hash_test.go +++ b/hash/hash_test.go @@ -2,6 +2,7 @@ package hash import ( "bytes" + "encoding/base64" "os" "testing" @@ -71,33 +72,33 @@ func TestSum(t *testing.T) { } var ( - ogsha1 = squish.B64d(kayosSHA1) - ogsha256 = squish.B64d(kayosSHA256) - ogsha512 = squish.B64d(kayosSHA512) - ogmd5 = squish.B64d(kayosMD5) - newsha1 = Sum(TypeSHA1, []byte("kayos\n")) - newsha256 = Sum(TypeSHA256, []byte("kayos\n")) - newsha512 = Sum(TypeSHA512, []byte("kayos\n")) - newmd5 = Sum(TypeMD5, []byte("kayos\n")) + ogsha1, _ = base64.StdEncoding.DecodeString(kayosSHA1) + ogsha256, _ = base64.StdEncoding.DecodeString(kayosSHA256) + ogsha512, _ = base64.StdEncoding.DecodeString(kayosSHA512) + ogmd5, _ = base64.StdEncoding.DecodeString(kayosMD5) + newsha1 = Sum(TypeSHA1, []byte("kayos\n")) + newsha256 = Sum(TypeSHA256, []byte("kayos\n")) + newsha512 = Sum(TypeSHA512, []byte("kayos\n")) + newmd5 = Sum(TypeMD5, []byte("kayos\n")) ) if !bytes.Equal(newsha1, ogsha1) { - t.Fatalf("[sha1] wanted: %v, got %v", kayosSHA1, squish.B64e(newsha1)) + t.Fatalf("[sha1] wanted: %v, got %v", ogsha1, newsha1) } t.Logf("[sha1] success: %s", kayosSHA1) if !bytes.Equal(newsha256, ogsha256) { - t.Fatalf("[sha256] wanted: %v, got %v", kayosSHA256, squish.B64e(newsha256)) + t.Fatalf("[sha256] wanted: %v, got %v", ogsha256, newsha256) } t.Logf("[sha256] success: %s", kayosSHA256) if !bytes.Equal(newsha512, ogsha512) { - t.Fatalf("[sha512] wanted: %v, got %v", kayosSHA512, squish.B64e(newsha512)) + t.Fatalf("[sha512] wanted: %v, got %v", ogsha512, newsha512) } t.Logf("[sha512] success: %s", kayosSHA512) if !bytes.Equal(newmd5, ogmd5) { - t.Fatalf("[md5] wanted: %v, got %v", kayosMD5, squish.B64e(newmd5)) + t.Fatalf("[md5] wanted: %v, got %v", ogmd5, newmd5) } t.Logf("[md5] success: %s", kayosMD5) } diff --git a/squish/squish.go b/squish/squish.go index 0f6275b..86a3e46 100644 --- a/squish/squish.go +++ b/squish/squish.go @@ -4,46 +4,94 @@ import ( "bytes" "compress/gzip" "encoding/base64" + "errors" "io" + "sync" +) + +var ( + bufPool = &sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, + } + + gzipPool = &sync.Pool{ + New: func() interface{} { + return gzip.NewWriter(nil) + }, + } ) // Gzip compresses as slice of bytes using gzip compression. func Gzip(data []byte) []byte { - var b bytes.Buffer - gz := gzip.NewWriter(&b) - // In theory this should never fail, and I don't know how to make the gzip buffered reader fail in testing. - _, _ = gz.Write(data) - _ = gz.Close() - return b.Bytes() + buf := bufPool.Get().(*bytes.Buffer) + gz := gzipPool.Get().(*gzip.Writer) + buf.Reset() + r, w := io.Pipe() + gz.Reset(w) + go func() { + _, _ = gz.Write(data) + _ = gz.Close() + _ = w.Close() + }() + n, _ := buf.ReadFrom(r) + buf.Truncate(int(n)) + _ = r.Close() + res, _ := io.ReadAll(buf) + bufPool.Put(buf) + gzipPool.Put(gz) + return res } // Gunzip decompresses a gzip compressed slice of bytes. func Gunzip(data []byte) (out []byte, err error) { - var gz *gzip.Reader - gz, err = gzip.NewReader(bytes.NewReader(data)) + gz, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { - return + return nil, err } - return io.ReadAll(gz) + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + var n int64 + n, _ = buf.ReadFrom(gz) + err = gz.Close() + buf.Truncate(int(n)) + res, _ := io.ReadAll(buf) + bufPool.Put(buf) + return res, err } // B64e encodes the given slice of bytes into base64 standard encoding. -func B64e(cytes []byte) (data string) { - data = base64.StdEncoding.EncodeToString(cytes) - return +func B64e(in []byte) (out string) { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + buf.Grow(base64.StdEncoding.EncodedLen(len(in))) + b64 := base64.NewEncoder(base64.StdEncoding, buf) + _, _ = b64.Write(in) + _ = b64.Close() + res := buf.Bytes() + bufPool.Put(buf) + return string(res) } // B64d decodes the given string into the original slice of bytes. // Do note that this is for non critical tasks, it has no error handling for purposes of clean code. func B64d(str string) (data []byte) { + if len(str) == 0 { + return nil + } data, _ = base64.StdEncoding.DecodeString(str) return data } // UnpackStr UNsafely unpacks (usually banners) that have been base64'd and then gzip'd. func UnpackStr(encoded string) (string, error) { - dcytes, err := Gunzip(B64d(encoded)) - if err != nil { + one := B64d(encoded) + if len(one) == 0 { + return "", errors.New("0 length base64 decoding result") + } + dcytes, err := Gunzip(one) + if err != nil && !errors.Is(err, io.EOF) { return "", err } return string(dcytes), nil diff --git a/squish/squish_test.go b/squish/squish_test.go index 0808825..b5a93c4 100644 --- a/squish/squish_test.go +++ b/squish/squish_test.go @@ -2,7 +2,10 @@ package squish import ( "bytes" + "encoding/base64" "testing" + + "git.tcp.direct/kayos/common/entropy" ) const lip string = ` @@ -32,7 +35,7 @@ func TestGzip(t *testing.T) { t.Logf("[PASS] Gzip compress succeeded, squished %d bytes.", profit) hosDown, err := Gunzip(gsUp) if err != nil { - t.Fatalf("Gzip decompression failed: %e", err) + t.Fatalf("Gzip decompression failed: %s", err.Error()) } if !bytes.Equal(hosDown, []byte(lip)) { t.Fatalf("[FAIL] Gzip decompression failed, data does not appear to be the same after decompression") @@ -46,14 +49,114 @@ func TestGzip(t *testing.T) { t.Fatalf("[FAIL] Gunzip didn't fail on nil input") } } -func TestUnpackStr(t *testing.T) { - packed := B64e(Gzip([]byte(lip))) + +func TestGunzipMustFails(t *testing.T) { + blank := "" + _, err := Gunzip([]byte(blank)) + if err == nil { + t.Fatalf("[FAIL] Gunzip didn't fail on empty input") + } + _, err = UnpackStr(blank) + if err == nil { + t.Fatalf("[FAIL] UnpackStr didn't fail on empty input") + } + junk := "junk" + _, err = Gunzip([]byte(junk)) + if err == nil { + t.Fatalf("[FAIL] Gunzip didn't fail on junk input") + } + _, err = UnpackStr(junk) + if err == nil { + t.Fatalf("[FAIL] UnpackStr didn't fail on junk input") + } +} + +func TestGzipEntropic(t *testing.T) { + for i := 0; i < 50; i++ { + dat := []byte(entropy.RandStr(entropy.RNG(55) * 1024)) + for len(dat) < 1024 { + dat = []byte(entropy.RandStr(entropy.RNG(55) * 1024)) + } + gzTest(dat, t) + } +} + +func gzTest(dat []byte, t *testing.T) { + t.Logf("Testing Gzip on %d bytes of data", len(dat)) + gsUp := Gzip(dat) + if bytes.Equal(gsUp, dat) { + t.Fatalf("[FAIL] Gzip didn't change the data at all despite being error free...") + } + if len(gsUp) == len(dat) || len(gsUp) > len(dat) { + t.Fatalf("[FAIL] Gzip didn't change the sise of the data at all (or it grew)... before: %d after: %d", + len(dat), len(gsUp)) + } + if len(gsUp) == 0 { + t.Fatalf("[FAIL] ended up with 0 bytes after compression...") + } + profit := len(dat) - len(gsUp) + t.Logf("[PASS] Gzip compress succeeded, squished %d bytes.", profit) + hosDown, err := Gunzip(gsUp) + if err != nil { + t.Fatalf("Gzip decompression failed: %s", err.Error()) + } + if !bytes.Equal(hosDown, dat) { + t.Fatalf("[FAIL] Gzip decompression failed, data does not appear to be the same after decompression") + } + if len(hosDown) != len(dat) { + t.Fatalf("[FAIL] Gzip decompression failed, data [%d] does not appear to be the same [%d] length after decompression", hosDown, len(dat)) + } + t.Logf("[PASS] Gzip decompress succeeded, restored %d bytes.", profit) +} + +func TestGzipDeterministic(t *testing.T) { + packed := Gzip([]byte(lip)) + for n := 0; n < 10; n++ { + again := Gzip([]byte(lip)) + if !bytes.Equal(again, packed) { + t.Fatalf("[FAIL] Gzip is not deterministic") + } + } +} + +func TestUnpackStr(t *testing.T) { //nolint:cyclop + gzd := Gzip([]byte(lip)) + if len(gzd) == 0 { + t.Fatalf("[FAIL] Gzip failed to compress data") + } + gzdSanity, gzdErr := Gunzip(gzd) + if gzdErr != nil { + t.Fatalf("Gzip failed: %s", gzdErr.Error()) + } + if !bytes.Equal(gzdSanity, []byte(lip)) { + t.Fatalf("Bytes not equal after Gzip: %v != %v", gzdSanity, []byte(lip)) + } + packed := B64e(gzd) + if len(packed) == 0 { + t.Fatalf("[FAIL] B64e failed to encode data") + } + t.Logf("Packed: %s", packed) + sanity1, err1 := base64.StdEncoding.DecodeString(packed) + if err1 != nil { + t.Fatalf("b64 failed: %s", err1.Error()) + } + if !bytes.Equal(sanity1, gzd) { + t.Fatalf("Bytes not equal after b64: %v != %v", sanity1, gzd) + } + sanity2, err2 := Gunzip(sanity1) + if err2 != nil { + t.Fatalf("Gzip failed: %s", err2.Error()) + } + if !bytes.Equal(sanity2, []byte(lip)) { + t.Fatalf("Bytes not equal after Gzip: %v != %v", sanity2, []byte(lip)) + } unpacked, err := UnpackStr(packed) switch { case err != nil: - t.Fatalf("[FAIL] %e", err) + t.Errorf("[FAIL] %s", err.Error()) case unpacked != lip: - t.Fatalf("[FAIL] unpackstr decided to not work, who knows why. If you see this than I have already become a janitor.") + t.Fatalf("[FAIL] unpackstr decided to not work, who knows why. If you see this than I have already become a janitor.\n"+ + "unpacked: %s != packed: %s", unpacked, lip) default: t.Logf("[PASS] TestUnpackStr") }