atmos
/
nym-directory
mirror of https://github.com/nymtech/nym-directory
1
0
Fork 0
Code Issues Releases Wiki Activity

Initial work on batch reporting

This commit is contained in:
jstuczyn 2020-10-07 12:32:16 +01:00
parent ac876afe39
commit 1ca202b92a
8 changed files with 179 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -20,6 +20,7 @@ require (
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/onsi/ginkgo v1.14.1
github.com/onsi/gomega v1.10.1
github.com/pkg/errors v0.9.1 // indirect
github.com/stretchr/testify v1.5.1
github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14
github.com/swaggo/gin-swagger v1.2.0

2
go.sum
View File

@ -274,6 +274,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=

51
mixmining/controller.go
View File

@ -10,31 +10,36 @@ import (
// Config for this controller
type Config struct {
Sanitizer Sanitizer
Service IService
BatchSanitizer BatchSanitizer
Sanitizer Sanitizer
Service IService
}
// controller is the metrics controller
// controller is the mixmining controller
type controller struct {
service IService
sanitizer Sanitizer
service IService
sanitizer Sanitizer
batchSanitizer BatchSanitizer
}
// Controller ...
type Controller interface {
CreateMixStatus(c *gin.Context)
RegisterRoutes(router *gin.Engine)
// TODO: ADD BATCH!!
}
// New returns a new mixmining.Controller
func New(cfg Config) Controller {
return &controller{cfg.Service, cfg.Sanitizer}
return &controller{cfg.Service, cfg.Sanitizer, cfg.BatchSanitizer}
}
func (controller *controller) RegisterRoutes(router *gin.Engine) {
router.POST("/api/mixmining", controller.CreateMixStatus)
router.GET("/api/mixmining/:pubkey/history", controller.ListMeasurements)
router.GET("/api/mixmining/:pubkey/report", controller.GetMixStatusReport)
router.POST("/api/mixmining/batch", controller.BatchCreateMixStatus)
// router.GET("/api/mixmining/fullreport", controller.BatchGetMixStatusReport) // TODO
}
// ListMeasurements lists mixnode statuses
@ -108,3 +113,37 @@ func (controller *controller) GetMixStatusReport(c *gin.Context) {
}
c.JSON(http.StatusOK, report)
}
// BatchCreateMixStatus ...
// @Summary Lets the network monitor create a new uptime status for multiple mixes
// @Description Nym network monitor sends packets through the system and checks if they make it. The network monitor then hits this method to report whether nodes were up at a given time.
// @ID addBatchMixStatus
// @Accept json
// @Produce json
// @Tags mixmining
// @Param object body models.BatchMixStatus true "object"
// @Success 201
// @Failure 400 {object} models.Error
// @Failure 404 {object} models.Error
// @Failure 500 {object} models.Error
// @Router /api/mixmining/batch [post]
func (controller *controller) BatchCreateMixStatus(c *gin.Context) {
remoteIP := strings.Split((c.Request.RemoteAddr), ":")[0]
if remoteIP != "127.0.0.1" {
c.JSON(http.StatusForbidden, gin.H{"error": "forbidden"})
return
}
var status models.BatchMixStatus
if err := c.ShouldBindJSON(&status); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
println("MADE IT HERE")
sanitized := controller.batchSanitizer.Sanitize(status)
persisted := controller.service.BatchCreateMixStatus(sanitized)
controller.service.SaveBatchStatusReport(persisted)
// persisted := controller.service.CreateMixStatus(sanitized)
// controller.service.SaveStatusReport(persisted)
// c.JSON(http.StatusCreated, gin.H{"ok": true})
}

30
mixmining/db.go
View File

@ -17,10 +17,13 @@ var DB *gorm.DB
// IDb holds status information
type IDb interface {
Add(models.PersistedMixStatus)
BatchAdd(status []models.PersistedMixStatus)
List(pubkey string, limit int) []models.PersistedMixStatus
ListDateRange(pubkey string, ipVersion string, start int64, end int64) []models.PersistedMixStatus
LoadReport(pubkey string) models.MixStatusReport
BatchLoadReports(pubkeys []string) models.BatchMixStatusReport
SaveMixStatusReport(models.MixStatusReport)
SaveBatchMixStatusReport(models.BatchMixStatusReport)
}
// Db is a hashtable that holds mixnode uptime mixmining
@ -61,6 +64,10 @@ func (db *Db) Add(status models.PersistedMixStatus) {
db.orm.Create(status)
}
func(db *Db) BatchAdd(status []models.PersistedMixStatus) {
db.orm.Create(status)
}
// List returns all models.PersistedMixStatus in the orm
func (db *Db) List(pubkey string, limit int) []models.PersistedMixStatus {
var statuses []models.PersistedMixStatus
@ -89,6 +96,18 @@ func (db *Db) SaveMixStatusReport(report models.MixStatusReport) {
}
}
func (db *Db) SaveBatchMixStatusReport(report models.BatchMixStatusReport) {
fmt.Printf("\r\nAbout to save batch report\r\n: %+v", report)
if result := db.orm.Save(report.Report); result.Error != nil {
fmt.Printf("Batch Mix status report save error: %+v", result.Error)
}
fmt.Errorf("DID IT WORK? SAVE BATCH")
}
// LoadReport retrieves a models.MixStatusReport.
// If a report ins't found, it crudely generates a new instance and returns that instead.
func (db *Db) LoadReport(pubkey string) models.MixStatusReport {
@ -100,3 +119,14 @@ func (db *Db) LoadReport(pubkey string) models.MixStatusReport {
}
return report
}
func (db *Db) BatchLoadReports(pubkeys []string) models.BatchMixStatusReport {
var reports []models.MixStatusReport
if retrieve := db.orm.Where("pub_key IN ?", pubkeys).Find(&reports); retrieve.Error != nil {
fmt.Printf("ERROR while retrieving multiple mix status report %+v", retrieve.Error)
return models.BatchMixStatusReport{Report: make([]models.MixStatusReport, 0)}
}
return models.BatchMixStatusReport{Report: reports}
}

69
mixmining/mocks/IDb.go
View File

@ -1,69 +0,0 @@
// Code generated by mockery v0.0.0-dev. DO NOT EDIT.
package mocks
import (
models "github.com/nymtech/nym-directory/models"
mock "github.com/stretchr/testify/mock"
)
// IDb is an autogenerated mock type for the IDb type
type IDb struct {
mock.Mock
}
// Add provides a mock function with given fields: _a0
func (_m *IDb) Add(_a0 models.PersistedMixStatus) {
_m.Called(_a0)
}
// List provides a mock function with given fields: pubkey, limit
func (_m *IDb) List(pubkey string, limit int) []models.PersistedMixStatus {
ret := _m.Called(pubkey, limit)
var r0 []models.PersistedMixStatus
if rf, ok := ret.Get(0).(func(string, int) []models.PersistedMixStatus); ok {
r0 = rf(pubkey, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]models.PersistedMixStatus)
}
}
return r0
}
// ListDateRange provides a mock function with given fields: pubkey, ipVersion, start, end
func (_m *IDb) ListDateRange(pubkey string, ipVersion string, start int64, end int64) []models.PersistedMixStatus {
ret := _m.Called(pubkey, ipVersion, start, end)
var r0 []models.PersistedMixStatus
if rf, ok := ret.Get(0).(func(string, string, int64, int64) []models.PersistedMixStatus); ok {
r0 = rf(pubkey, ipVersion, start, end)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]models.PersistedMixStatus)
}
}
return r0
}
// LoadReport provides a mock function with given fields: pubkey
func (_m *IDb) LoadReport(pubkey string) models.MixStatusReport {
ret := _m.Called(pubkey)
var r0 models.MixStatusReport
if rf, ok := ret.Get(0).(func(string) models.MixStatusReport); ok {
r0 = rf(pubkey)
} else {
r0 = ret.Get(0).(models.MixStatusReport)
}
return r0
}
// SaveMixStatusReport provides a mock function with given fields: _a0
func (_m *IDb) SaveMixStatusReport(_a0 models.MixStatusReport) {
_m.Called(_a0)
}

29
mixmining/sanitizer.go
View File

@ -5,7 +5,34 @@ import (
"github.com/nymtech/nym-directory/models"
)
// Sanitizer sanitizes untrusted metrics data. It should be used in
// BatchSanitizer sanitizes untrusted batch mixmining data. It should be used in
// controllers to wipe out any questionable input at our application's front
// door.
type BatchSanitizer interface {
Sanitize(input models.BatchMixStatus) models.BatchMixStatus
}
type batchSanitizer struct {
sanitizer sanitizer
}
// NewSanitizer returns a new input sanitizer for metrics
func NewBatchSanitizer(policy *bluemonday.Policy) batchSanitizer {
return batchSanitizer {
sanitizer: sanitizer {
policy: policy,
},
}
}
func (s batchSanitizer) Sanitize(input models.BatchMixStatus) models.BatchMixStatus {
for i := range input.Status {
input.Status[i] = s.sanitizer.Sanitize(input.Status[i])
}
return input
}
// Sanitizer sanitizes untrusted mixmining data. It should be used in
// controllers to wipe out any questionable input at our application's front
// door.
type Sanitizer interface {

70
mixmining/service.go
View File

@ -12,12 +12,16 @@ type Service struct {
db IDb
}
// IService defines the REST service interface for metrics.
// IService defines the REST service interface for mixmining.
type IService interface {
CreateMixStatus(metric models.MixStatus) models.PersistedMixStatus
CreateMixStatus(mixStatus models.MixStatus) models.PersistedMixStatus
List(pubkey string) []models.PersistedMixStatus
SaveStatusReport(status models.PersistedMixStatus) models.MixStatusReport
GetStatusReport(pubkey string) models.MixStatusReport
SaveBatchStatusReport(status []models.PersistedMixStatus) models.BatchMixStatusReport
BatchCreateMixStatus(batchMixStatus models.BatchMixStatus) []models.PersistedMixStatus
BatchGetMixStatusReport() models.BatchMixStatusReport
}
// NewService constructor
@ -47,11 +51,54 @@ func (service *Service) GetStatusReport(pubkey string) models.MixStatusReport {
return service.db.LoadReport(pubkey)
}
// SaveStatusReport builds and saves a status report for a mixnode. The report can be updated once
// whenever we receive a new status, and the saved result can then be queried. This keeps us from
// having to build the report dynamically on every request at runtime.
func (service *Service) SaveStatusReport(status models.PersistedMixStatus) models.MixStatusReport {
report := service.db.LoadReport(status.PubKey)
func (service *Service) BatchCreateMixStatus(batchMixStatus models.BatchMixStatus) []models.PersistedMixStatus {
statusList := make([]models.PersistedMixStatus, len(batchMixStatus.Status))
for i, mixStatus := range batchMixStatus.Status {
persistedMixStatus := models.PersistedMixStatus{
MixStatus: mixStatus,
Timestamp: timemock.Now().UnixNano(),
}
statusList[i] = persistedMixStatus
}
service.db.BatchAdd(statusList)
return statusList
}
func (service *Service) BatchGetMixStatusReport() models.BatchMixStatusReport {
return models.BatchMixStatusReport{}
}
func (service *Service) SaveBatchStatusReport(status []models.PersistedMixStatus) models.BatchMixStatusReport {
pubkeys := make([]string, len(status))
for i := range status {
pubkeys[i] = status[i].PubKey
}
batchReport := service.db.BatchLoadReports(pubkeys)
// that's super crude but I don't think db results are guaranteed to come in order, plus some entries might
// not exist
reportMap := make(map[string]int)
for i, report := range batchReport.Report {
reportMap[report.PubKey] = i
}
for _, mixStatus := range status {
if reportIdx, ok := reportMap[mixStatus.PubKey]; ok {
service.DealWithStatusReport(&batchReport.Report[reportIdx], &mixStatus)
} else {
var freshReport models.MixStatusReport
service.DealWithStatusReport(&freshReport, &mixStatus)
batchReport.Report = append(batchReport.Report, freshReport)
}
}
service.db.SaveBatchMixStatusReport(batchReport)
return batchReport
}
func (service *Service) DealWithStatusReport(report *models.MixStatusReport, status *models.PersistedMixStatus) {
report.PubKey = status.PubKey // crude, we do this in case it's a fresh struct returned from the db
if status.IPVersion == "4" {
@ -69,6 +116,15 @@ func (service *Service) SaveStatusReport(status models.PersistedMixStatus) model
report.LastWeekIPV6 = service.CalculateUptime(status.PubKey, "6", daysAgo(7))
report.LastMonthIPV6 = service.CalculateUptime(status.PubKey, "6", daysAgo(30))
}
}
// SaveStatusReport builds and saves a status report for a mixnode. The report can be updated once
// whenever we receive a new status, and the saved result can then be queried. This keeps us from
// having to build the report dynamically on every request at runtime.
func (service *Service) SaveStatusReport(status models.PersistedMixStatus) models.MixStatusReport {
report := service.db.LoadReport(status.PubKey)
service.DealWithStatusReport(&report, &status)
service.db.SaveMixStatusReport(report)
return report
}

10
models/mixmining.go
View File

@ -40,3 +40,13 @@ type MixStatusReport struct {
LastWeekIPV6 int `json:"lastWeekIPV6" binding:"required"`
LastMonthIPV6 int `json:"lastMonthIPV6" binding:"required"`
}
// BatchMixStatus allows to indicate whether given set of nodes is up or down, as reported by a Nym monitor node.
type BatchMixStatus struct {
Status []MixStatus `json:"status" binding:"required"`
}
// BatchMixStatusReport gives a quick view of network uptime performance
type BatchMixStatusReport struct {
Report []MixStatusReport `json:"report" binding:"required"`
}