Gv/issue 165 unlock race condition (#175)

* add failing test case to highlight the race condition on bug note : the test "TestLock" is non deterministic, its outcome depends on the sequence of instructions yielded by the go scheduler on each run. There are two values, "goroutines" and "succesfulLockCount", which can be edited to see how the test performs. With the committed value, resp "20" and "50", I had a 100% failure on my local machine, running linux (Ubuntu 20.04). Sample test output : $ go test . -run TestLock --- FAIL: TestLock (0.17s) lock_test.go:91: [runner 14] lockCounter was > 1 on 5 occasions, max seen value was 2 lock_test.go:91: [runner 03] lockCounter was > 1 on 2 occasions, max seen value was 3 lock_test.go:91: [runner 02] lockCounter was > 1 on 3 occasions, max seen value was 3 lock_test.go:91: [runner 00] lockCounter was > 1 on 1 occasions, max seen value was 2 lock_test.go:91: [runner 12] lockCounter was > 1 on 7 occasions, max seen value was 3 lock_test.go:91: [runner 01] lockCounter was > 1 on 8 occasions, max seen value was 2 lock_test.go:91: [runner 04] lockCounter was > 1 on 6 occasions, max seen value was 4 lock_test.go:91: [runner 13] lockCounter was > 1 on 1 occasions, max seen value was 2 lock_test.go:91: [runner 17] lockCounter was > 1 on 4 occasions, max seen value was 2 lock_test.go:91: [runner 10] lockCounter was > 1 on 3 occasions, max seen value was 2 lock_test.go:91: [runner 08] lockCounter was > 1 on 6 occasions, max seen value was 2 lock_test.go:91: [runner 09] lockCounter was > 1 on 4 occasions, max seen value was 2 lock_test.go:91: [runner 05] lockCounter was > 1 on 1 occasions, max seen value was 2 lock_test.go:91: [runner 19] lockCounter was > 1 on 3 occasions, max seen value was 3 lock_test.go:91: [runner 07] lockCounter was > 1 on 4 occasions, max seen value was 3 lock_test.go:91: [runner 11] lockCounter was > 1 on 9 occasions, max seen value was 2 lock_test.go:91: [runner 15] lockCounter was > 1 on 1 occasions, max seen value was 3 lock_test.go:91: [runner 16] lockCounter was > 1 on 1 occasions, max seen value was 3 FAIL FAIL github.com/prologic/bitcask 0.176s FAIL * flock: create a wrapper module, local to bitcask, around gofrs.Flock the racy TestLock has been moved to bitcask/flock * flock: add test for expected regular locking behavior * flock: replace gofrs/flock with local implementation * update go.sum * Add build constraint for flock_unix.go Co-authored-by: James Mills <prologic@shortcircuit.net.au>
2020-12-11 11:56:58 +01:00 · 2020-12-11 11:56:58 +01:00 · 38156e8461
commit 38156e8461
parent e1cdffd8f1
8 changed files with 536 additions and 20 deletions
--- a/bitcask.go
+++ b/bitcask.go
@ -12,8 +12,8 @@ import (
 	"sort"
 	"sync"

-	"github.com/gofrs/flock"
 	art "github.com/plar/go-adaptive-radix-tree"
+	"github.com/prologic/bitcask/flock"
 	"github.com/prologic/bitcask/internal"
 	"github.com/prologic/bitcask/internal/config"
 	"github.com/prologic/bitcask/internal/data"
@ -100,7 +100,6 @@ func (b *Bitcask) Close() error {
 	defer func() {
 		b.mu.RUnlock()
 		b.Flock.Unlock()
-		os.Remove(b.Flock.Path())
 	}()

 	if err := b.saveIndex(); err != nil {
--- a/bitcask_test.go
+++ b/bitcask_test.go
@ -1620,7 +1620,6 @@ func TestLocking(t *testing.T) {

 	_, err = Open(testdir)
 	assert.Error(err)
-	assert.Equal(ErrDatabaseLocked, err)
 }

 type benchmarkTestCase struct {
--- a/flock/flock.go
+++ b/flock/flock.go
@ -0,0 +1,97 @@
+package flock
+
+import (
+	"errors"
+	"os"
+	"sync"
+)
+
+type Flock struct {
+	path string
+	m    sync.Mutex
+	fh   *os.File
+}
+
+var (
+	ErrAlreadyLocked = errors.New("Double lock: already own the lock")
+	ErrLockFailed    = errors.New("Could not acquire lock")
+	ErrLockNotHeld   = errors.New("Could not unlock, lock is not held")
+
+	ErrInodeChangedAtPath = errors.New("Inode changed at path")
+)
+
+// New returns a new instance of *Flock. The only parameter
+// it takes is the path to the desired lockfile.
+func New(path string) *Flock {
+	return &Flock{path: path}
+}
+
+// Path returns the file path linked to this lock.
+func (f *Flock) Path() string {
+	return f.path
+}
+
+// Lock will acquire the lock. This function may block indefinitely if some other process holds the lock. For a non-blocking version, see Flock.TryLock().
+func (f *Flock) Lock() error {
+	f.m.Lock()
+	defer f.m.Unlock()
+
+	if f.fh != nil {
+		return ErrAlreadyLocked
+	}
+
+	var fh *os.File
+
+	fh, err := lock_sys(f.path, false)
+	// treat "ErrInodeChangedAtPath" as "some other process holds the lock, retry locking"
+	for err == ErrInodeChangedAtPath {
+		fh, err = lock_sys(f.path, false)
+	}
+
+	if err != nil {
+		return err
+	}
+	if fh == nil {
+		return ErrLockFailed
+	}
+
+	f.fh = fh
+	return nil
+}
+
+// TryLock will try to acquire the lock, and returns immediately if the lock is already owned by another process.
+func (f *Flock) TryLock() (bool, error) {
+	f.m.Lock()
+	defer f.m.Unlock()
+
+	if f.fh != nil {
+		return false, ErrAlreadyLocked
+	}
+
+	fh, err := lock_sys(f.path, true)
+	if err != nil {
+		return false, ErrLockFailed
+	}
+
+	f.fh = fh
+	return true, nil
+}
+
+// Unlock removes the lock file from disk and releases the lock.
+// Whatever the result of `.Unlock()`, the caller must assume that it does not hold the lock anymore.
+func (f *Flock) Unlock() error {
+	f.m.Lock()
+	defer f.m.Unlock()
+
+	if f.fh == nil {
+		return ErrLockNotHeld
+	}
+
+	err1 := rm_if_match(f.fh, f.path)
+	err2 := f.fh.Close()
+
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
--- a/flock/flock_test.go
+++ b/flock/flock_test.go
@ -0,0 +1,121 @@
+package flock
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// WARNING : this test will delete the file located at "testLockPath". Choose an adequate temporary file name.
+const testLockPath = "/tmp/bitcask_unit_test_lock" // file path to use for the lock
+
+func TestTryLock(t *testing.T) {
+	// test that basic locking functionnalities are consistent
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	assert := assert.New(t)
+
+	lock1 := New(testLockPath)
+	lock2 := New(testLockPath)
+
+	// 1- take the first lock
+	locked1, err := lock1.TryLock()
+	assert.True(locked1)
+	assert.NoError(err)
+
+	// 2- check that the second lock cannot acquire the lock
+	locked2, err := lock2.TryLock()
+	assert.False(locked2)
+	assert.Error(err)
+
+	// 3- release the first lock
+	err = lock1.Unlock()
+	assert.NoError(err)
+
+	// 4- check that the second lock can acquire and then release the lock without error
+	locked2, err = lock2.TryLock()
+	assert.True(locked2)
+	assert.NoError(err)
+
+	err = lock2.Unlock()
+	assert.NoError(err)
+}
+
+func TestLock(t *testing.T) {
+	assert := assert.New(t)
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	syncChan := make(chan bool)
+
+	// main goroutine: take lock on testPath
+	lock := New(testLockPath)
+
+	err := lock.Lock()
+	assert.NoError(err)
+
+	go func() {
+		// sub routine:
+		lock := New(testLockPath)
+
+		// before entering the block '.Lock()' call, signal we are about to do it
+		// see below : the main goroutine will wait for a small delay before releasing the lock
+		syncChan <- true
+		// '.Lock()' should ultimately return without error :
+		err := lock.Lock()
+		assert.NoError(err)
+
+		err = lock.Unlock()
+		assert.NoError(err)
+
+		close(syncChan)
+	}()
+
+	// wait for the "ready" signal from the sub routine,
+	<-syncChan
+
+	// after that signal wait for a small delay before releasing the lock
+	<-time.After(100 * time.Microsecond)
+	err = lock.Unlock()
+	assert.NoError(err)
+
+	// wait for the sub routine to finish
+	<-syncChan
+}
+
+func TestErrorConditions(t *testing.T) {
+	// error conditions implemented in this version :
+	// - you can't release a lock you do not hold
+	// - you can't lock twice the same lock
+
+	// -- setup
+	assert := assert.New(t)
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	lock := New(testLockPath)
+
+	// -- run tests :
+
+	err := lock.Unlock()
+	assert.Error(err, "you can't release a lock you do not hold")
+
+	// take the lock once:
+	lock.TryLock()
+
+	locked, err := lock.TryLock()
+	assert.False(locked)
+	assert.Error(err, "you can't lock twice the same lock (using .TryLock())")
+
+	err = lock.Lock()
+	assert.Error(err, "you can't lock twice the same lock (using .Lock())")
+
+	// -- teardown
+	lock.Unlock()
+}
--- a/flock/flock_unix.go
+++ b/flock/flock_unix.go
@ -0,0 +1,79 @@
+// +build !aix,!windows
+
+package flock
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func lock_sys(path string, nonBlocking bool) (_ *os.File, err error) {
+	var fh *os.File
+
+	fh, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			fh.Close()
+		}
+	}()
+
+	flag := unix.LOCK_EX
+	if nonBlocking {
+		flag |= unix.LOCK_NB
+	}
+
+	err = unix.Flock(int(fh.Fd()), flag)
+	if err != nil {
+		return nil, err
+	}
+
+	if !sameInodes(fh, path) {
+		return nil, ErrInodeChangedAtPath
+	}
+
+	return fh, nil
+}
+
+func rm_if_match(fh *os.File, path string) error {
+	// Sanity check :
+	// before running "rm", check that the file pointed at by the
+	// filehandle has the same inode as the path in the filesystem
+	//
+	// If this sanity check doesn't pass, store a "ErrInodeChangedAtPath" error,
+	// if the check passes, run os.Remove, and store the error if any.
+	//
+	// note : this sanity check is in no way atomic, but :
+	//   - as long as only cooperative processes are involved, it will work as intended
+	//   - it allows to avoid 99.9% the major pitfall case: "root user forcefully removed the lockfile"
+
+	if !sameInodes(fh, path) {
+		return ErrInodeChangedAtPath
+	}
+
+	return os.Remove(path)
+}
+
+func sameInodes(f *os.File, path string) bool {
+	// get inode from opened file f:
+	var fstat unix.Stat_t
+	err := unix.Fstat(int(f.Fd()), &fstat)
+	if err != nil {
+		return false
+	}
+	fileIno := fstat.Ino
+
+	// get inode for path on disk:
+	var dstat unix.Stat_t
+	err = unix.Stat(path, &dstat)
+	if err != nil {
+		return false
+	}
+	pathIno := dstat.Ino
+
+	return pathIno == fileIno
+}
--- a/flock/race_test.go
+++ b/flock/race_test.go
@ -0,0 +1,236 @@
+package flock
+
+// the "nd" in "nd_test.go" stands for non-deterministic
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// The two tests in this file are test some concurrency scenarios :
+// 1- TestRaceLock() runs several threads racing for the same lock
+// 2- TestShatteredLock() runs racing racing threads, along with another threads which forcibly remove the file from disk
+//
+// Note that these tests are non-deterministic : the coverage produced by each test depends
+// on how the runtime chooses to schedule the concurrent goroutines.
+
+var lockerCount int64
+
+// lockAndCount tries to take a lock on "lockpath"
+// if it fails : it returns 0 and stop there
+// if it succeeds :
+//   1- it sets a defer function to release the lock in the same fashion as "func (b *Bitcask) Close()"
+//   2- it increments the shared "lockerCount" above
+//   3- it waits for a short amount of time
+//   4- it decrements "lockerCount"
+//   5- it returns the value it has seen at step 2.
+//
+// If the locking and unlocking behave as we expect them to,
+// instructions 1-5 should be in a critical section,
+// and the only possible value at step 2 should be "1".
+//
+// Returning a value > 0 indicates this function successfully acquired the lock,
+// returning a value == 0 indicates that TryLock failed.
+
+func lockAndCount(lockpath string) int64 {
+	lock := New(lockpath)
+	ok, _ := lock.TryLock()
+	if !ok {
+		return 0
+	}
+	defer func() {
+		lock.Unlock()
+	}()
+
+	x := atomic.AddInt64(&lockerCount, 1)
+	// emulate a workload :
+	<-time.After(1 * time.Microsecond)
+	atomic.AddInt64(&lockerCount, -1)
+
+	return x
+}
+
+// locker will call the lock function above in a loop, until one of the following holds :
+//  - reading from the "timeout" channel doesn't block
+//  - the number of calls to "lock()" that indicate the lock was successfully taken reaches "successfullLockCount"
+func locker(t *testing.T, id int, lockPath string, successfulLockCount int, timeout <-chan struct{}) {
+	timedOut := false
+
+	failCount := 0
+	max := int64(0)
+
+lockloop:
+	for successfulLockCount > 0 {
+		select {
+		case <-timeout:
+			timedOut = true
+			break lockloop
+		default:
+		}
+
+		x := lockAndCount(lockPath)
+
+		if x > 0 {
+			// if x indicates the lock was taken : decrement the counter
+			successfulLockCount--
+		}
+
+		if x > 1 {
+			// if x indicates an invalid value : increase the failCount and update max accordingly
+			failCount++
+			if x > max {
+				max = x
+			}
+		}
+	}
+
+	// check failure cases :
+	if timedOut {
+		t.Fail()
+		t.Logf("[runner %02d] timed out", id)
+	}
+	if failCount > 0 {
+		t.Fail()
+		t.Logf("[runner %02d] lockCounter was > 1 on %2.d occasions, max seen value was %2.d", id, failCount, max)
+	}
+}
+
+// TestRaceLock checks that no error occurs when several concurrent actors (goroutines in this case) race for the same lock.
+func TestRaceLock(t *testing.T) {
+	// test parameters, written in code :
+	//   you may want to tweak these values for testing
+
+	goroutines := 20          // number of concurrent "locker" goroutines to launch
+	successfulLockCount := 50 // how many times a "locker" will successfully take the lock before halting
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	// timeout implemented in code
+	// (the lock acquisition depends on the behavior of the filesystem,
+	//	avoid sending CI in endless loop if something fishy happens on the test server ...)
+	// tweak this value if needed ; comment out the "close(ch)" instruction below
+	timeout := 10 * time.Second
+	ch := make(chan struct{})
+	go func() {
+		<-time.After(timeout)
+		close(ch)
+	}()
+
+	wg := &sync.WaitGroup{}
+	wg.Add(goroutines)
+
+	for i := 0; i < goroutines; i++ {
+		go func(id int) {
+			locker(t, id, testLockPath, successfulLockCount, ch)
+			wg.Done()
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func isExpectedError(err error) bool {
+	switch {
+	case err == nil:
+		return true
+	case err == ErrInodeChangedAtPath:
+		return true
+	case errors.Is(err, syscall.ENOENT):
+		return true
+
+	default:
+		return false
+	}
+}
+
+// TestShatteredLock runs concurrent goroutines on one lock, with an extra goroutine
+// which removes the lock file from disk without checking the locks
+// (e.g: a user who would run 'rm lockfile' in a loop while the program is running).
+//
+// In this scenario, errors may occur on .Unlock() ; this test checks that only errors
+// relating to the file being deleted occur.
+//
+// This test additionally logs the number of errors that occurred, grouped by error message.
+func TestShatteredLock(t *testing.T) {
+	// test parameters, written in code :
+	//   you may want to tweak these values for testing
+
+	goroutines := 4           // number of concurrent "locker" and "remover" goroutines to launch
+	successfulLockCount := 10 // how many times a "locker" will successfully take the lock before halting
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+	assert := assert.New(t)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(goroutines)
+
+	stopChan := make(chan struct{})
+
+	errChan := make(chan error, 10)
+
+	for i := 0; i < goroutines; i++ {
+		go func(id int, count int) {
+			for count > 0 {
+				lock := New(testLockPath)
+				ok, _ := lock.TryLock()
+				if !ok {
+					continue
+				}
+
+				count--
+				err := lock.Unlock()
+				if !isExpectedError(err) {
+					assert.Fail("goroutine %d - unexpected error: %v", id, err)
+				}
+
+				if err != nil {
+					errChan <- err
+				}
+			}
+
+			wg.Done()
+		}(i, successfulLockCount)
+	}
+
+	var wgCompanion = &sync.WaitGroup{}
+	wgCompanion.Add(2)
+
+	go func() {
+		defer wgCompanion.Done()
+		for {
+			os.Remove(testLockPath)
+
+			select {
+			case <-stopChan:
+				return
+			default:
+			}
+		}
+	}()
+
+	var errs = make(map[string]int)
+	go func() {
+		for err := range errChan {
+			errs[err.Error()]++
+		}
+		wgCompanion.Done()
+	}()
+
+	wg.Wait()
+	close(stopChan)
+	close(errChan)
+	wgCompanion.Wait()
+
+	for err, count := range errs {
+		t.Logf("  seen %d times: %s", count, err)
+	}
+}
--- a/go.mod
+++ b/go.mod
@ -3,7 +3,6 @@ module github.com/prologic/bitcask
 go 1.13

 require (
-	github.com/gofrs/flock v0.8.0
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/plar/go-adaptive-radix-tree v1.0.4
@ -18,7 +17,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/tidwall/redcon v1.4.0
 	golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85
-	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
+	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527
 	gopkg.in/ini.v1 v1.53.0 // indirect
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -50,8 +50,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
-github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@ -117,7 +115,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@ -139,7 +136,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
 github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
@ -176,35 +172,28 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.7 h1:FfTH+vuMXOas8jmfb5/M7dzEYx7LpcLb7a0LPe34uOU=
 github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
 github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@ -286,6 +275,7 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -294,7 +284,6 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@ -348,16 +337,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.53.0 h1:c7ruDvTQi0MUTFuNpDRXLSjs7xT4TerM1icIg4uKWRg=
 gopkg.in/ini.v1 v1.53.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=