From 494b4e75b8bed15f0101e6200c2e51cf1a4301d7 Mon Sep 17 00:00:00 2001 From: "kayos@tcp.direct" Date: Mon, 15 Feb 2021 11:59:12 -0800 Subject: [PATCH] working towards disappearing images ad txt bin + cleanup --- config.go | 62 ------------ config.toml | 6 +- db.go | 27 ------ img.go | 264 ---------------------------------------------------- main.go | 50 ---------- router.go | 58 ------------ util.go | 12 --- 7 files changed, 3 insertions(+), 476 deletions(-) delete mode 100644 config.go delete mode 100644 db.go delete mode 100644 img.go delete mode 100644 main.go delete mode 100644 router.go delete mode 100644 util.go diff --git a/config.go b/config.go deleted file mode 100644 index cc91292..0000000 --- a/config.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "fmt" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/spf13/viper" - "strconv" -) - -///////////////////////////////// -func configRead() { - viper.SetConfigName("config") // filename without ext - viper.SetConfigType("toml") // also defines extension - - viper.AddConfigPath("/etc/tcpac/") // multiple possible - viper.AddConfigPath(".") // locations for config - - err = viper.ReadInConfig() - if err != nil { // this should be replaced with more intelligent handling - panic(fmt.Errorf("Fatal error reading config file: %s \n", err)) - } - - //// fetch config directives from file //// - debugBool = viper.GetBool("global.debug") // we need to load the debug boolean first - // so we can output config directives - if debugBool { - zerolog.SetGlobalLevel(zerolog.DebugLevel) - log.Debug().Msg("Debug mode enabled") - } else { - zerolog.SetGlobalLevel(zerolog.InfoLevel) - } - - s = "http.baseurl" - baseUrl = viper.GetString(s) - log.Debug().Str(s, baseUrl).Msg("configRead()") - - s = "http.port" - i := viper.GetInt(s) - webPort = strconv.Itoa(i) // int looks cleaner in config - log.Debug().Str(s, webPort).Msg("configRead()") // but we reference it as a string later - - s = "http.bindip" - webIP = viper.GetString(s) - log.Debug().Str(s, webIP).Msg("configRead()") - - s = "files.data" - dbDir = viper.GetString(s) - log.Debug().Str(s, dbDir).Msg("configRead()") // where we're actually gonna store everything - - s = "files.logs" - logDir = viper.GetString(s) - log.Debug().Str(s, logDir).Msg("configRead()") - - s = "img.uidsize" - uidSize = viper.GetInt(s) - log.Debug().Int(s, uidSize).Msg("configRead()") - - s = "img.delkeysize" - keySize = viper.GetInt(s) - log.Debug().Int(s, keySize).Msg("configRead()") -} diff --git a/config.toml b/config.toml index f603009..f8042fe 100644 --- a/config.toml +++ b/config.toml @@ -1,7 +1,7 @@ title = "tcp.ac config" [global] -debug = true +debug = false [http] baseurl = "http://127.0.0.1:8080/" @@ -13,5 +13,5 @@ uidsize = 6 delkeysize = 24 [files] -data = "./data/" -logs = "./logs/" +data = "../data/" +logs = "../logs/" diff --git a/db.go b/db.go deleted file mode 100644 index 26886a4..0000000 --- a/db.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "github.com/prologic/bitcask" - "github.com/rs/zerolog/log" -) - -func dbInit() { - opts := []bitcask.Option{ - bitcask.WithMaxValueSize(24 / 1024 / 1024), - } - - hashDB, _ = bitcask.Open(dbDir+"hsh", opts...) // this will probably only be for images? - log.Debug().Msg("Initializing checksum database") - - keyDB, _ = bitcask.Open(dbDir+"key", opts...) // delete keys (maybe for all objects?) - log.Debug().Msg("Initializing key database") - - imgDB, _ = bitcask.Open(dbDir+"img", opts...) // literal image files - log.Debug().Msg("Initializing img database") - - txtDB, _ = bitcask.Open(dbDir + "txt") // pastebin - log.Debug().Msg("Initializing txt database") - - urlDB, _ = bitcask.Open(dbDir + "url") // url shortener entries - log.Debug().Msg("Initializing url database") -} diff --git a/img.go b/img.go deleted file mode 100644 index d1fcc2b..0000000 --- a/img.go +++ /dev/null @@ -1,264 +0,0 @@ -package main - -import ( - "bytes" - valid "github.com/asaskevich/govalidator" - "github.com/gin-gonic/gin" - "github.com/rs/zerolog/log" - exifremove "github.com/scottleedavis/go-exif-remove" - "github.com/twharmon/gouid" - "golang.org/x/crypto/blake2b" - "image" - _ "image/gif" - "io" - "io/ioutil" - "net/http" - "strings" -) - -var fExt string - -type Post struct { - Imgurl string `json:"Imgurl"` - Delkey string `json:"Delkey"` -} - -func postUpload(c *gin.Context, id string, key string) { - imgurl := baseUrl + "i/" + string(id) - keyurl := "duplicate" - if key != "nil" { - keyurl = baseUrl + "d/i/" + string(key) - } - - log.Info().Str("func", "imgPost").Str("id", id).Str("status", "201").Str("imgurl", imgurl).Str("keyurl", keyurl) - c.JSON(201, gin.H{"delkey": keyurl, "imgurl": imgurl}) - return -} - -func imgDel(c *gin.Context) { - fn = "imgDel" - - log.Debug().Str("func", fn).Msg("Request received!") // received request - rKey := c.Param("key") - if len(rKey) != 16 || !valid.IsAlphanumeric(rKey) { - log.Error().Str("func", fn).Msg("delete request failed sanity check!") - errThrow(c, 400, "400", "400") - return - } - - targetImg, _ := keyDB.Get([]byte(rKey)) - if targetImg == nil || !strings.Contains(string(targetImg), "i.") { - log.Error().Str("func", fn).Str("rkey", rKey).Msg("no img delete entry found with provided key") - errThrow(c, 400, "400", "400") - return - } - - finalTarget := strings.Split(string(targetImg), ".") - - if !imgDB.Has([]byte(finalTarget[1])) { - log.Error().Str("func", fn).Str("rkey", rKey).Msg("corresponding image not found in database!") - errThrow(c, 500, "500", "500") // this shouldn't happen...? - return - } - err := imgDB.Delete([]byte(finalTarget[1])) - if err != nil { - log.Error().Str("func", fn).Str("rkey", finalTarget[1]).Msg("delete failed!") - errThrow(c, 500, "500", "500") - return - } - - if imgDB.Has([]byte(finalTarget[1])) { - log.Error().Str("func", fn).Str("rkey", finalTarget[1]).Msg("delete failed!?") - errThrow(c, 500, "500", "500") - return - } - - log.Info().Str("func", fn).Str("rkey", finalTarget[1]).Msg("Image file deleted successfully") - log.Debug().Str("func", fn).Str("rkey", finalTarget[1]).Msg("Removing delete key entry") - err = keyDB.Delete([]byte(rKey)) - if err != nil { - log.Error().Str("func", fn).Str("rkey", finalTarget[1]).Msg("Couldn't delete key") - // it would be insane to try and delete the hash here - } // if someone is uploading this image again after del - c.JSON(200, "DELETE_SUCCESS") // and the file corresponding to the hash no longer exists - // we will delete the hash entry then and re-add then -} - -func imgView(c *gin.Context) { - fn = "imgView" - // the user can access their image with or without a file extension in URI - log.Debug().Str("func", fn).Msg("request received") // however it must be a valid extension (more checks further down) - sUid := strings.Split(c.Param("uid"), ".") - rUid := sUid[0] - if len(sUid) > 1 { - fExt = strings.ToLower(sUid[1]) - log.Debug().Str("func", fn).Str("ext", fExt).Msg("detected file extension") - if fExt != "png" && fExt != "jpg" && fExt != "jpeg" && fExt != "gif" { - log.Error().Str("func", fn).Msg("Bad file extension!") - errThrow(c, 400, "400", "400") - return - } - } else { - fExt = "nil" - } - - if !valid.IsAlphanumeric(rUid) || len(rUid) < 3 || len(rUid) > 16 { - log.Error().Str("func", fn).Msg("request discarded as invalid") // these limits should be variables eventually - errThrow(c, 400, "400", "400") - return - } - - log.Debug().Str("func", fn).Str("rUid", rUid).Msg("request validated") // now that we think its a valid request we will query - - fBytes, _ := imgDB.Get([]byte(rUid)) - if fBytes == nil { - log.Error().Str("func", fn).Str("rUid", rUid).Msg("no corresponding file for this id") - errThrow(c, 404, "404", "File not found") - return - } - - file := bytes.NewReader(fBytes) - imageFormat, ok := checkImage(file) - if !ok { - errThrow(c, http.StatusBadRequest, "400", "400") - log.Error().Str("func", fn).Str("rUid", rUid).Msg("the file we grabbed is not an image!?") // not sure how a non image would get uploaded - return // however, better safe than sorry - } else { - log.Debug().Str("func", fn).Str("rUid", rUid).Str("imageFormat", imageFormat).Msg("Image format detected") - } - - if fExt != "nil" && fExt != imageFormat { // additional extension sanity check - log.Error().Str("func", fn).Str("rUid", rUid).Msg("requested file extension does not match filetype") - errThrow(c, 400, "400", "400") - return - } - - contentType := "image/" + imageFormat // extension or not - // we give them the proper content type - c.Data(200, contentType, fBytes) - - log.Info().Str("func", fn).Str("rUid", rUid).Msg("Successful upload") -} - -func imgPost(c *gin.Context) { - fn = "imgPost" - - var Scrubbed []byte - - f, err := c.FormFile("upload") - if err != nil { - errThrow(c, http.StatusBadRequest, err.Error(), "400") // 400 bad request - } // incoming POST data is invalid - - log.Debug().Str("func", fn).Str("filename", f.Filename).Msg("[+] New upload") - - file, err := f.Open() - if err != nil { - errThrow(c, http.StatusInternalServerError, err.Error(), "error processing file\n") - } - - log.Debug().Str("func", fn).Msg("verifying file is an image") - imageFormat, ok := checkImage(file) - if !ok { - errThrow(c, http.StatusBadRequest, "400", "input does not appear to be an image") - return - } else { - log.Debug().Str("func", fn).Msg("image file type detected") - } - - log.Debug().Str("func", fn).Msg("dumping byte form of file") - fbytes, err := ioutil.ReadAll(file) - if imageFormat != "gif" { - log.Debug().Str("func", fn).Err(err).Msg("error scrubbing exif") - Scrubbed, err = exifremove.Remove(fbytes) - if err != nil { - errThrow(c, http.StatusInternalServerError, err.Error(), "error scrubbing exif") - return - } - } else { - log.Debug().Str("func", fn).Msg("skipping exif scrub for gif image") - Scrubbed = fbytes - } - - log.Debug().Str("func", fn).Msg("calculating blake2b checksum") - - Hashr, _ := blake2b.New(64, nil) - Hashr.Write(Scrubbed) - hash := Hashr.Sum(nil) - - log.Debug().Str("func", fn).Msg("Checking for duplicate's in database") - - imgRef, _ := hashDB.Get(hash) - ogUid := string(imgRef) - - if imgRef != nil { - log.Debug().Str("func", fn).Str("ogUid", ogUid).Msg("duplicate checksum in hash database, checking if file still exists...") - if imgDB.Has(imgRef) { - log.Debug().Str("func", fn).Str("ogUid", ogUid).Msg("duplicate file found! returning original URL") - postUpload(c, ogUid, "nil") // they weren't the original uploader so they don't get a delete key - return - } else { - log.Debug().Str("func", fn).Str("ogUid", ogUid).Msg("stale hash found, deleting entry...") - hashDB.Delete(hash) - } - } - - log.Info().Str("func", fn).Msg("no duplicate images found, generating uid and delete key") - - uid := gouid.String(uidSize) // these should both be config directives eventually - key := gouid.String(keySize) // generate delete key - - // lets make sure that we don't clash even though its highly unlikely - for uidRef, _ := imgDB.Get([]byte(uid)); uidRef != nil; { - log.Info().Str("func", fn).Msg(" uid already exists! generating new...") - uid = gouid.String(uidSize) - } - for keyRef, _ := keyDB.Get([]byte(key)); keyRef != nil; { - log.Info().Str("func", fn).Msg(" delete key already exists! generating new...") - key = gouid.String(keySize) - } - - hashDB.Put([]byte(hash), []byte(uid)) // save checksum to db to prevent dupes in the future - - log.Debug().Str("func", fn).Str("uid", uid).Msg("saving file to database") - - err = imgDB.Put([]byte(uid), []byte(Scrubbed)) - if err != nil { - errThrow(c, 500, err.Error(), "upload failed") - return - } - - err = keyDB.Put([]byte(key), []byte("i."+uid)) // add delete key to database with image prefix - if err != nil { - errThrow(c, http.StatusInternalServerError, err.Error(), "internal error") - return - } - - log.Debug().Str("func", fn).Str("uid", uid).Msg("saved to database successfully, sending to postUpload") - - postUpload(c, uid, key) - -} - -func checkImage(r io.ReadSeeker) (string, bool) { - _, fmt, err := image.Decode(r) - _, err2 := r.Seek(0, 0) - if err != nil || err2 != nil { - return "", false - } - return fmt, true -} - -func getSize(s io.Seeker) (size int64, err error) { - if _, err = s.Seek(0, 0); err != nil { - return - } - - // 2 == from the end of the file - if size, err = s.Seek(0, 2); err != nil { - return - } - - _, err = s.Seek(0, 0) - return -} diff --git a/main.go b/main.go deleted file mode 100644 index b060fb7..0000000 --- a/main.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "github.com/rs/zerolog/log" - "github.com/rs/zerolog" - "os" -) - -func init() { - // initialize the logger before the config: that way we can output debug lines - // pertaining to the parsing of the configuration init - - //////////// init logging //////////// - - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) // before we read config, do Stderr pretty print (need logs location) - - log.Info().Msg("Initializing...") - - // see config.go - configRead() - - // now that we know where to put the log file, we can start output (replace logger) - - err = os.MkdirAll(logDir, 0755) - if err != nil { - log.Fatal().Str("directory",logDir).Str("intent","logDir").Err(err) - return - } - - err = os.MkdirAll(dbDir, 0755) - if err != nil { - log.Fatal().Str("directory",dbDir).Str("intent","dbDir").Err(err) - return - } - - lf, err := os.OpenFile(logDir+"tcpac.log", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666) - if err != nil { - log.Fatal().Str("logDir",logDir).Err(err).Msg("Error opening log file!") - } - - consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout} - multi := zerolog.MultiLevelWriter(consoleWriter, lf) - log.Logger = zerolog.New(multi).With().Timestamp().Logger() - dbInit() -} - -func main() { - // see router.go - httpRouter() -} diff --git a/router.go b/router.go deleted file mode 100644 index 1c1ef8b..0000000 --- a/router.go +++ /dev/null @@ -1,58 +0,0 @@ - -package main - -import ( - "github.com/gin-contrib/logger" - "github.com/gin-contrib/gzip" - "github.com/gin-gonic/gin" -) - -func txtPost(c *gin.Context) { - return -} - -func urlPost(c *gin.Context) { - return -} - -func httpRouter() { - router := gin.New() - - router.MaxMultipartMemory = 16 << 20 // crude POST limit (fix this) - - // use gzip compression unless someone requests something with an explicit extension - router.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedPathsRegexs([]string{".*"}))) - - router.Use(logger.SetLogger()) // use our own logger - - // static html and such - // workaround the issue where the router tries to handle /* - router.Static("/h", "./public") - router.StaticFile("/favicon.ico", "./public/favicon.ico") - router.GET("/", func(c *gin.Context) { c.Redirect(301, "h/") }) - - - imgR := router.Group("/i") - { - imgR.GET("/", func(c *gin.Context) { c.String(200,"") }) // javascript wants something here idk - imgR.POST("/put", imgPost) // put looks nicer even though its actually POST - imgR.GET("/:uid", imgView) - } - - delR := router.Group("/d") - { - delR.GET("/i/:key", imgDel) - } - - txtR := router.Group("/t") - { - txtR.POST("/put", txtPost) - } - - urlR := router.Group("/u") - { - urlR.POST("/put", urlPost) - } - - router.Run(webIP + ":" + webPort) -} diff --git a/util.go b/util.go deleted file mode 100644 index 9a65ebf..0000000 --- a/util.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -import ( - "github.com/gin-gonic/gin" -) - -func errThrow(c *gin.Context, respcode int, Error string, msg string) { - // log.Error().Str("IP",c.ClientIP()).Str("err",Error).Msg(msg) - if debugBool { - c.String(respcode, msg) - } -}