kayos
/
common
1
0
Fork 0
Code Issues Pull Requests Projects Releases 11 Wiki Activity

entropy: fix race condition, massive performance boost (#17) 

* Fix[entropy]: Implement rand pool to remediate splitmix64 race condition

* Perf[entropy]: Use recycled byte buffers for random strings

* Refactor[entropy][testing][bench]: clean up test code and report allocations

* Refactor[entropy]: nil out rand pointer after we put it back into the pool

* Testing[entropy]: fix coverage

* Fix[entropy]: Fix race condition during testing conditions

* gomod: retract premature tags
This commit is contained in:
kayos 2023-09-06 23:45:44 -07:00 committed by GitHub
parent ba99ee5f4c
commit cb6e9c71a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 171 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
entropy/entropy.go
View File

@ -8,13 +8,53 @@ import (
"time"
"nullprogram.com/x/rng"
"git.tcp.direct/kayos/common/pool"
)
type randPool struct {
sync.Pool
}
func (p *randPool) Get() *rand.Rand {
return p.Pool.Get().(*rand.Rand)
}
func (p *randPool) Put(r *rand.Rand) {
p.Pool.Put(r)
}
var (
lolXD = randPool{
Pool: sync.Pool{
New: func() interface{} {
sm64 := new(rng.SplitMix64)
sm64.Seed(GetCryptoSeed())
prng := rand.New(sm64) //nolint:gosec
return prng
},
},
}
hardLocc = &sync.Mutex{}
sharedRand *rand.Rand
getSharedRand = &sync.Once{}
)
func setSharedRand() {
hardLocc.Lock()
sharedRand = lolXD.Get()
hardLocc.Unlock()
}
func AcquireRand() *rand.Rand {
return lolXD.Get()
}
func ReleaseRand(r *rand.Rand) {
lolXD.Put(r)
r = nil
}
// RandomStrChoice returns a random item from an input slice of strings.
func RandomStrChoice(choice []string) string {
if len(choice) > 0 {
@ -24,50 +64,51 @@ func RandomStrChoice(choice []string) string {
}
// GetCryptoSeed returns a random int64 derived from crypto/rand.
// This can be used as a seed for the math/rand package.
// This can be used as a seed for various PRNGs.
func GetCryptoSeed() int64 {
var seed int64
_ = binary.Read(crip.Reader, binary.BigEndian, &seed)
return seed
}
// GetOptimizedRand returns a pointer to a *new* rand.Rand which uses crypto/rand to seed a splitmix64 rng.
// GetOptimizedRand returns a pointer to a *new* rand.Rand which uses GetCryptoSeed to seed an rng.SplitMix64.
// Does not use the global/shared instance of a splitmix64 rng, but instead creates a new one.
func GetOptimizedRand() *rand.Rand {
r := new(rng.SplitMix64)
r.Seed(GetCryptoSeed())
return rand.New(r)
return rand.New(r) //nolint:gosec
}
// GetSharedRand returns a pointer to our shared optimized rand.Rand which uses crypto/rand to seed a splitmix64 rng.
// WARNING - RACY - This is not thread safe, and should only be used in a single-threaded context.
func GetSharedRand() *rand.Rand {
getSharedRand.Do(func() {
sharedRand = GetOptimizedRand()
setSharedRand()
})
return sharedRand
}
// RNGUint32 returns a random uint32 using crypto/rand and splitmix64.
func RNGUint32() uint32 {
getSharedRand.Do(func() {
sharedRand = GetOptimizedRand()
})
return sharedRand.Uint32()
r := lolXD.Get()
ui := r.Uint32()
lolXD.Put(r)
return ui
}
/*
RNG returns integer with a maximum amount of 'n' using a global/shared instance of a splitmix64 rng.
- Benchmark_FastRandStr5-24 25205089 47.03 ns/op
- Benchmark_FastRandStr5-24 25205089 47.03 ns/op
- Benchmark_FastRandStr25-24 7113620 169.8 ns/op
- Benchmark_FastRandStr55-24 3520297 340.7 ns/op
- Benchmark_FastRandStr500-24 414966 2837 ns/op
- Benchmark_FastRandStr55555-24 3717 315229 ns/op
*/
func RNG(n int) int {
getSharedRand.Do(func() {
sharedRand = GetOptimizedRand()
})
return sharedRand.Intn(n)
r := lolXD.Get()
i := r.Intn(n)
lolXD.Put(r)
return i
}
// OneInA generates a random number with a maximum of 'million' (input int).
@ -88,22 +129,54 @@ func RandSleepMS(n int) {
const charset = "abcdefghijklmnopqrstuvwxyz1234567890"
const charsetWithUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890"
var strBufs = pool.NewBufferFactory()
// RandStr generates a random alphanumeric string with a max length of size.
// Alpha charset used is a-z all lowercase.
func RandStr(size int) string {
buf := make([]byte, size)
for i := 0; i != size; i++ {
buf[i] = charset[GetOptimizedRand().Uint32()%uint32(len(charset))]
}
return string(buf)
return randStr(false, size)
}
// RandStrWithUpper generates a random alphanumeric string with a max length of size.
// Alpha charset used is a-Z mixed case.
func RandStrWithUpper(size int) string {
buf := make([]byte, size)
for i := 0; i != size; i++ {
buf[i] = charsetWithUpper[uint32(RNG(62))%uint32(len(charsetWithUpper))]
}
return string(buf)
return randStr(true, size)
}
/*
randStr is an overoptimized (read: dummy fast) random string generator.
using byte buffers gives us a solid 2 alloc/op indefinitely.
using string builders gives us a linear increase in alloc/op, up to 22 alloc/op at 55,555 characters.
at 55,555 characters, we are at:
~57,000 bytes per op with string builders
vs
~210,000 bytes per op with string builders.
this is felt significantly at ranges as low as 500 chars, where we get:
8 alloc/op and >1,000 bytes/op with string builders
vs
2 alloc/op and ~500 bytes/op with byte buffers.
*/
func randStr(upper bool, size int) string {
buf := strBufs.Get()
r := lolXD.Get()
for i := 0; i != size; i++ {
ui32 := int(r.Uint32())
switch upper {
case true:
_ = buf.WriteByte(charsetWithUpper[ui32%len(charsetWithUpper)])
case false:
_ = buf.WriteByte(charset[ui32%len(charset)])
}
}
lolXD.Put(r)
s := buf.String()
strBufs.MustPut(buf)
return s
}

142
entropy/entropy_test.go
View File

@ -1,6 +1,7 @@
package entropy
import (
"fmt"
"strings"
"sync"
"testing"
@ -8,7 +9,8 @@ import (
var dupCount = 0
func check[T comparable](zero T, one T, t *testing.T) {
func check[T comparable](t *testing.T, zero T, one T) {
t.Helper()
if zero == one {
dupCount++
t.Errorf("hit a duplicate! %v == %v", zero, one)
@ -17,11 +19,14 @@ func check[T comparable](zero T, one T, t *testing.T) {
}
func Test_RNG(t *testing.T) {
t.Parallel()
// for coverage
sharedRand = GetSharedRand()
setSharedRand()
RandSleepMS(5)
hardLocc.Lock()
sharedRand = nil
getSharedRand = &sync.Once{}
hardLocc.Unlock()
// - - - - - -
if OneInA(1000000) {
println(string([]byte{
@ -31,13 +36,41 @@ func Test_RNG(t *testing.T) {
}))
}
for n := 0; n != 500; n++ {
check(RNG(55555), RNG(55555), t)
check(RNGUint32(), RNGUint32(), t)
for n := 0; n != 55555; n++ {
check(t, RNG(123454321), RNG(123454321))
check(t, RNGUint32(), RNGUint32())
}
// for coverage
if GetOptimizedRand().Intn(55555) == GetOptimizedRand().Intn(55555) {
t.Errorf("GetOptimizedRand(55555) returned the same value twice!")
}
if GetSharedRand().Intn(55555) == GetSharedRand().Intn(55555) {
t.Errorf("GetSharedRand(55555) returned the same value twice!")
}
r := AcquireRand()
one := r.Intn(55555)
two := r.Intn(55555)
if one == two {
t.Errorf("AcquireRand() returned the same value twice!")
}
ReleaseRand(r)
r = AcquireRand()
one1 := r.Intn(55555)
two1 := r.Intn(55555)
if one1 == two1 {
t.Errorf("AcquireRand() returned the same value twice!")
}
if one == one1 {
t.Errorf("AcquireRand()[2] returned the same value twice!")
}
if two == two1 {
t.Errorf("AcquireRand()[2] returned the same value twice!")
}
}
func Test_OneInA(t *testing.T) {
t.Parallel()
for n := 0; n < 100; n++ {
yes := ""
if OneInA(1) {
@ -49,42 +82,48 @@ func Test_OneInA(t *testing.T) {
}
}
func randStrChecks(zero, one string, t *testing.T, intendedLength int) {
func randStrChecks(t *testing.T, zero, one string, intendedLength int) {
t.Helper()
if len(zero) != len(one) {
t.Fatalf("RandStr output length inconsistency, len(zero) is %d but wanted len(one) which is %d", len(zero), len(one))
}
if len(zero) != intendedLength || len(one) != intendedLength {
t.Fatalf("RandStr output length inconsistency, len(zero) is %d and len(one) is %d, but both should have been 55", len(zero), len(one))
t.Fatalf(
"RandStr output length inconsistency, "+
"len(zero) is %d and len(one) is %d, but both should have been 55", len(zero), len(one))
}
check(zero, one, t)
check(t, zero, one)
}
func Test_RandStr(t *testing.T) {
t.Parallel()
for n := 0; n != 500; n++ {
zero := RandStr(55)
one := RandStr(55)
t.Logf("Random0: %s Random1: %s", zero, one)
randStrChecks(zero, one, t, 55)
randStrChecks(t, zero, one, 55)
}
t.Logf("[SUCCESS] RandStr had no collisions")
}
func Test_RandStrWithUpper(t *testing.T) {
t.Parallel()
for n := 0; n != 500; n++ {
zero := RandStrWithUpper(15)
one := RandStrWithUpper(15)
t.Logf("Random0: %s Random1: %s", zero, one)
randStrChecks(zero, one, t, 15)
randStrChecks(t, zero, one, 15)
}
t.Logf("[SUCCESS] RandStr had no collisions")
}
func Test_RandStr_Entropy(t *testing.T) {
t.Parallel()
var totalScore = 0
for n := 0; n != 500; n++ {
zero := RandStr(55)
one := RandStr(55)
randStrChecks(zero, one, t, 55)
randStrChecks(t, zero, one, 55)
zeroSplit := strings.Split(zero, "")
oneSplit := strings.Split(one, "")
var similarity = 0
@ -93,10 +132,10 @@ func Test_RandStr_Entropy(t *testing.T) {
continue
}
similarity++
// t.Logf("[-] zeroSplit[%d] is the same as oneSplit[%d] (%s)", i, i, char)
}
if similarity*4 > 55 {
t.Errorf("[ENTROPY FAILURE] more than a quarter of the string is the same!\n zero: %s \n one: %s \nTotal similar: %d",
t.Errorf("[ENTROPY FAILURE] more than a quarter of the string is the same!\n "+
"zero: %s \n one: %s \nTotal similar: %d",
zero, one, similarity)
}
// t.Logf("[ENTROPY] Similarity score (lower is better): %d", similarity)
@ -106,6 +145,7 @@ func Test_RandStr_Entropy(t *testing.T) {
}
func Test_RandomStrChoice(t *testing.T) {
t.Parallel()
if RandomStrChoice([]string{}) != "" {
t.Fatalf("RandomStrChoice returned a value when given an empty slice")
}
@ -113,72 +153,30 @@ func Test_RandomStrChoice(t *testing.T) {
for n := 0; n != 500; n++ {
slice = append(slice, RandStr(555))
}
check(RandomStrChoice(slice), RandomStrChoice(slice), t)
check(t, RandomStrChoice(slice), RandomStrChoice(slice))
}
func Test_RNGUint32(t *testing.T) {
t.Parallel()
// start globals fresh, just for coverage.
sharedRand = GetOptimizedRand()
setSharedRand()
hardLocc.Lock()
getSharedRand = &sync.Once{}
hardLocc.Unlock()
RNGUint32()
}
func Benchmark_RandStr5(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStr(5)
}
}
func Benchmark_RandStr25(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStr(25)
}
}
func Benchmark_RandStr55(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStr(55)
}
}
func Benchmark_RandStr500(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStr(500)
}
}
func Benchmark_RandStr55555(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStr(55555)
}
}
func Benchmark_RandStrWithUpper5(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStrWithUpper(5)
}
}
func Benchmark_RandStrWithUpper25(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStrWithUpper(25)
}
}
func Benchmark_RandStrWithUpper55(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStrWithUpper(55)
}
}
func Benchmark_RandStrWithUpper500(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStrWithUpper(500)
}
}
func Benchmark_RandStrWithUpper55555(b *testing.B) {
for n := 0; n != b.N; n++ {
RandStrWithUpper(55555)
func Benchmark_RandStr(b *testing.B) {
toTest := []int{5, 25, 55, 500, 55555}
for n := range toTest {
for i := 1; i != 5; i++ {
b.Run(fmt.Sprintf("lenSeries%d/run%d", n, i), func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for tn := 0; tn != b.N; tn++ {
RandStr(n)
}
})
}
}
}

6
go.mod
View File

@ -9,4 +9,8 @@ require (
require golang.org/x/sys v0.11.0 // indirect
retract v0.0.0-20220210125455-40e3d2190a52
retract (
v0.0.0-20220210125455-40e3d2190a52
v0.9.0 // premature
v0.9.1 // premature (race condition)
)