adding index functionality, implementing gzip

config.go

@@ -1,40 +1,14 @@
 package main
 
 import (
-	"github.com/prologic/bitcask"
+	"fmt"
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
-	"github.com/rs/zerolog"
 	"strconv"
-	"fmt"
-	"os"
 )
 
-////////////// global declarations
-// datastores
-var imgDB      *bitcask.Bitcask
-var hashDB     *bitcask.Bitcask
-var keyDB      *bitcask.Bitcask
-var urlDB      *bitcask.Bitcask
-var txtDB      *bitcask.Bitcask
-// config directives
-var debugBool  bool
-var baseUrl    string
-var webPort    string
-var webIP      string
-var dbDir      string
-var logDir     string
-var uidSize    int
-var keySize    int
-// utilitarian globals
-var s	       string
-var fn	       string
-var i	       int
-var err        error
-var f          *os.File
 /////////////////////////////////
-
-
 func configRead() {
 	viper.SetConfigName("config") // filename without ext
 	viper.SetConfigType("toml")   //  also defines extension
@@ -51,8 +25,10 @@ func configRead() {
 	debugBool = viper.GetBool("global.debug") // we need to load the debug boolean first
 	//  so we can output config directives
 	if debugBool {
-		log.Debug().Msg("Debug mode enabled")
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+		log.Debug().Msg("Debug mode enabled")
+	} else {
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
 	}
 
 	s = "http.baseurl"

config.toml

@@ -1,7 +1,7 @@
 title = "tcp.ac config"
 
 [global]
-debug      = true
+debug      = false
 
 [http]
 baseurl    = "http://127.0.0.1:8080/"

db.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"github.com/prologic/bitcask"
 	"fmt"
+	"github.com/prologic/bitcask"
 )
 
-
 func dbInit() {
 	opts := []bitcask.Option{
 		bitcask.WithMaxValueSize(24 / 1024 / 1024),

img.go

@@ -1,19 +1,19 @@
 package main
 
 import (
+	"bytes"
 	valid "github.com/asaskevich/govalidator"
-	exifremove "github.com/scottleedavis/go-exif-remove"
-	"golang.org/x/crypto/blake2b"
-	"github.com/twharmon/gouid"
-	"github.com/rs/zerolog/log"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
+	exifremove "github.com/scottleedavis/go-exif-remove"
+	"github.com/twharmon/gouid"
+	"golang.org/x/crypto/blake2b"
+	"image"
 	_ "image/gif"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
-	"image"
-	"bytes"
-	"io"
 )
 
 var fExt string
@@ -26,7 +26,9 @@ type Post struct {
 func postUpload(c *gin.Context, id string, key string) {
 	imgurl := baseUrl + "i/" + string(id)
 	keyurl := "duplicate"
-	if key != "nil" { keyurl = baseUrl + "d/i/" + string(key) }
+	if key != "nil" {
+		keyurl = baseUrl + "d/i/" + string(key)
+	}
 
 	log.Info().Str("func", "imgPost").Str("id", id).Str("status", "201").Str("imgurl", imgurl).Str("keyurl", keyurl)
 	c.JSON(201, gin.H{"delkey": keyurl, "imgurl": imgurl})
@@ -38,14 +40,14 @@ func imgDel(c *gin.Context) {
 
 	log.Debug().Str("func", fn).Msg("Request received!") // received request
 	rKey := c.Param("key")
-	if (len(rKey) != 16 || !valid.IsAlphanumeric(rKey)) {
+	if len(rKey) != 16 || !valid.IsAlphanumeric(rKey) {
 		log.Error().Str("func", fn).Msg("delete request failed sanity check!")
 		errThrow(c, 400, "400", "400")
 		return
 	}
 
 	targetImg, _ := keyDB.Get([]byte(rKey))
-	if (targetImg == nil || !strings.Contains(string(targetImg), "i.")) {
+	if targetImg == nil || !strings.Contains(string(targetImg), "i.") {
 		log.Error().Str("func", fn).Str("rkey", rKey).Msg("no img delete entry found with provided key")
 		errThrow(c, 400, "400", "400")
 		return
@@ -82,7 +84,6 @@ func imgDel(c *gin.Context) {
 	// we will delete the hash entry then and re-add then
 }
 
-
 func imgView(c *gin.Context) {
 	fn = "imgView"
 	// the user can access their image with or without a file extension in URI
@@ -92,15 +93,16 @@ func imgView(c *gin.Context) {
 	if len(sUid) > 1 {
 		fExt = strings.ToLower(sUid[1])
 		log.Debug().Str("func", fn).Str("ext", fExt).Msg("detected file extension")
-		if (fExt != "png" && fExt != "jpg" && fExt != "jpeg" && fExt != "gif") {
+		if fExt != "png" && fExt != "jpg" && fExt != "jpeg" && fExt != "gif" {
 			log.Error().Str("func", fn).Msg("Bad file extension!")
 			errThrow(c, 400, "400", "400")
 			return
 		}
-	} else { fExt = "nil" }
+	} else {
+		fExt = "nil"
+	}
 
-
+	if !valid.IsAlphanumeric(rUid) || len(rUid) < 3 || len(rUid) > 16 {
-	if (!valid.IsAlphanumeric(rUid) || len(rUid) < 3 || len(rUid) > 16) {
 		log.Error().Str("func", fn).Msg("request discarded as invalid") // these limits should be variables eventually
 		errThrow(c, 400, "400", "400")
 		return
@@ -115,17 +117,17 @@ func imgView(c *gin.Context) {
 		return
 	}
 
-
-
 	file := bytes.NewReader(fBytes)
 	imageFormat, ok := checkImage(file)
 	if !ok {
 		errThrow(c, http.StatusBadRequest, "400", "400")
 		log.Error().Str("func", fn).Str("rUid", rUid).Msg("the file we grabbed is not an image!?") // not sure how a non image would get uploaded
 		return                                                                                     //  however, better safe than sorry
-	} else { log.Debug().Str("func",fn).Str("rUid",rUid).Str("imageFormat",imageFormat).Msg("Image format detected") }
+	} else {
+		log.Debug().Str("func", fn).Str("rUid", rUid).Str("imageFormat", imageFormat).Msg("Image format detected")
+	}
 
-	if (fExt != "nil" && fExt != imageFormat) {			// additional extension sanity check
+	if fExt != "nil" && fExt != imageFormat { // additional extension sanity check
 		log.Error().Str("func", fn).Str("rUid", rUid).Msg("requested file extension does not match filetype")
 		errThrow(c, 400, "400", "400")
 		return
@@ -138,7 +140,6 @@ func imgView(c *gin.Context) {
 	log.Info().Str("func", fn).Str("rUid", rUid).Msg("Successful upload")
 }
 
-
 func imgPost(c *gin.Context) {
 	fn = "imgPost"
 
@@ -161,7 +162,9 @@ func imgPost(c *gin.Context) {
 	if !ok {
 		errThrow(c, http.StatusBadRequest, "400", "input does not appear to be an image")
 		return
-	} else { log.Debug().Str("func",fn).Msg("image file type detected") }
+	} else {
+		log.Debug().Str("func", fn).Msg("image file type detected")
+	}
 
 	log.Debug().Str("func", fn).Msg("dumping byte form of file")
 	fbytes, err := ioutil.ReadAll(file)
@@ -205,18 +208,16 @@ func imgPost(c *gin.Context) {
 	uid := gouid.String(uidSize) // these should both be config directives eventually
 	key := gouid.String(keySize) // generate delete key
 
-
 	// lets make sure that we don't clash even though its highly unlikely
 	for uidRef, _ := imgDB.Get([]byte(uid)); uidRef != nil; {
 		log.Info().Str("func", fn).Msg(" uid already exists! generating new...")
-		uid = gouid.String(5)
+		uid = gouid.String(uidSize)
 	}
 	for keyRef, _ := keyDB.Get([]byte(key)); keyRef != nil; {
 		log.Info().Str("func", fn).Msg(" delete key already exists! generating new...")
-		key = gouid.String(16)
+		key = gouid.String(keySize)
 	}
 
-
 	hashDB.Put([]byte(hash), []byte(uid)) // save checksum to db to prevent dupes in the future
 
 	log.Debug().Str("func", fn).Str("uid", uid).Msg("saving file to database")

main.go

@@ -44,9 +44,6 @@ func init() {
 	consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout}
 	multi := zerolog.MultiLevelWriter(consoleWriter, lf)
 	log.Logger = zerolog.New(multi).With().Timestamp().Logger()
-
-	zerolog.SetGlobalLevel(zerolog.InfoLevel) // default is info and above
-
 	dbInit()
 }
 

router.go

@@ -1,6 +1,9 @@
+
 package main
 
 import (
+	"github.com/gin-contrib/logger"
+	"github.com/gin-contrib/gzip"
 	"github.com/gin-gonic/gin"
 )
 
@@ -15,7 +18,19 @@ func urlPost(c *gin.Context) {
 func httpRouter() {
 	router := gin.New()
 
-	router.MaxMultipartMemory = 16 << 20
+	router.MaxMultipartMemory = 16 << 20 // crude POST limit (fix this)
+
+	// use gzip compression unless someone requests something with an explicit extension
+	router.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedPathsRegexs([]string{".*"})))
+
+	router.Use(logger.SetLogger()) // use our own logger
+
+	// static html and such
+	// workaround the issue where the router tries to handle /*
+	router.Static("/h", "./public")
+	router.StaticFile("/favicon.ico", "./public/favicon.ico")
+	router.GET("/", func(c *gin.Context) { c.Redirect(301, "h/") })
+
 
 	imgR := router.Group("/i")
 	{

site/highlight.html

@@ -1,111 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-    <head>
-        <meta charset="UTF-8" />
-        <meta name="viewport" content="width=device-width, initial-scale=1" />
-        <title>{{ title }}</title>
-        <style>
-		            
-		.hljs {
-		  display: block;
-		  overflow-x: auto;
-		  padding: 0.5em;
-		  background: #000;
-		  color: #f8f8f8;
-		}
-
-		.hljs-comment,
-		.hljs-quote,
-		.hljs-meta {
-		  color: #7c7c7c;
-		}
-
-		.hljs-keyword,
-		.hljs-selector-tag,
-		.hljs-tag,
-		.hljs-name {
-		  color: #96cbfe;
-		}
-
-		.hljs-attribute,
-		.hljs-selector-id {
-		  color: #ffffb6;
-		}
-
-		.hljs-string,
-		.hljs-selector-attr,
-		.hljs-selector-pseudo,
-		.hljs-addition {
-		  color: #a8ff60;
-		}
-
-		.hljs-subst {
-		  color: #daefa3;
-		}
-
-		.hljs-regexp,
-		.hljs-link {
-		  color: #e9c062;
-		}
-
-		.hljs-title,
-		.hljs-section,
-		.hljs-type,
-		.hljs-doctag {
-		  color: #ffffb6;
-		}
-
-		.hljs-symbol,
-		.hljs-bullet,
-		.hljs-variable,
-		.hljs-template-variable,
-		.hljs-literal {
-		  color: #c6c5fe;
-		}
-
-		.hljs-number,
-		.hljs-deletion {
-		  color:#ff73fd;
-		}
-
-		.hljs-emphasis {
-		  font-style: italic;
-		}
-
-		.hljs-strong {
-		  font-weight: bold;
-		}        
-
-            html,
-            body {
-                overflow: hidden;
-            }
-            html,
-            body,
-            pre {
-                margin: 0;
-                padding: 0;
-            }
-            code {
-                position: absolute;
-                top: 0;
-                left: 0;
-                width: 100vw;
-                height: 100vh;
-                max-width: 100%;
-                max-height: 100%;
-                font-family: SFMono-Regular, Consolas, Liberation Mono, Menlo,
-                    monospace;
-            }
-        </style>
-    </head>
-    <body>
-        <pre><code>{{ contents }}</code></pre>
-        <script
-            src="js/highlight.min.js"></script>
-        {{ languages }}
-        <script>
-            hljs.initHighlightingOnLoad();
-        </script>
-    </body>
-</html>

site/index.html

@@ -1,385 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-    <head>
-        <meta charset="utf-8" />
-        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-        <link rel="stylesheet" href="css/spectre.min.css"/>
-        <link rel="stylesheet" href="css/spectre-icons.min.css"/>
-        <style>
-            div[id$="-form"]:not(.active) {
-                display: none;
-            }
-        </style>
-        <title>tcp.ac</title>
-<pre style="text-align: center; font-weight: bold;">
-  ,d                                                     
-  88                                                     
-MM88MMM ,adPPYba, 8b,dPPYba,      ,adPPYYba,  ,adPPYba,  
-  88   a8"     "" 88P'    "8a     ""     `Y8 a8"     ""  
-  88   8b         88       d8     ,adPPPPP88 8b          
-  88,  "8a,   ,aa 88b,   ,a8" 888 88,    ,88 "8a,   ,aa  
-  "Y888 `"Ybbd8"' 88`YbbdP"'  888 `"8bbdP"Y8  `"Ybbd8"'  
-                  88                                     
-                  88                                     
-
-</pre>
-    </head>
-    <body>
-        <nav class="container mb-2 pb-2">
-            <div class="columns">
-                <div
-                    class="column col-sm-12 col-md-10 col-lg-8 col-6 col-mx-auto"
-                >
-                    <ul class="tab tab-block">
-
-                        <li id="img-tab" class="tab-item active">
-                            <a href="#">img</a>
-                        </li>
-
-                        <li id="url-tab" class="tab-item" style="display:none;">  
-                            <a href="#">txt</a>
-                        </li>
-                        
-                        <li id="url-tab" class="tab-item" style="display:none;"> 
-                            <a href="#">url</a>
-                        </li>
-                    </ul>
-                </div>
-            </div>
-        </nav>
-        <main class="container mt-2 pt-2">
-            <div class="columns">
-                <div
-                    id="img-form"
-                    class="column col-sm-12 col-md-10 col-lg-8 col-6 col-mx-auto active">
-                    <div class="form-group">
-                        <label class="form-label" for="img-file">img</label>
-                        <input
-                            class="form-input"
-                            id="img-file"
-                            type="file"
-                            required/>
-                            <button
-                                class="btn btn-primary input-group-btn"
-                                id="img-submit"
-                                disabled>
-                                <i class="icon icon-upload"></i>
-                            </button>                            
-                        <p class="form-input-hint">accepted: jpeg,png,gif</p>                        
-                    </div>
-                </div>
-                <div
-                    id="url-form"
-                    class="column col-sm-12 col-md-10 col-lg-8 col-6 col-mx-auto">
-                    <div class="form-group">
-                        <label class="form-label" for="url-url">url</label>
-                        <div class="input-group">
-                            <span class="input-group-addon">/l/</span>
-                            <input
-                                id="url-url"
-                                class="form-input"
-                                type="text"
-                                placeholder="404040"
-                                required
-                            />
-                            <button
-                                class="btn btn-primary input-group-btn"
-                                id="url-submit"
-                                disabled
-                            >
-                                <i class="icon icon-upload"></i>
-                            </button>
-                        </div>
-                    </div>
-                    <div class="form-group">
-                        <label class="form-label" for="url-forward"
-                            >Forward</label
-                        >
-                        <input
-                            id="url-forward"
-                            class="form-input"
-                            type="url"
-                            placeholder="http://legitwebsite.cool/goatse.png"
-                            required
-                        />
-                    </div>
-                </div>
-                <div
-                    id="txt-form"
-                    class="column col-sm-12 col-md-10 col-lg-8 col-6 col-mx-auto"
-                >
-                    <div class="form-group">
-                        <label class="form-label" for="txt-url">url</label>
-                        <div class="input-group">
-                            <span class="input-group-addon">/t/</span>
-                            <input
-                                id="txt-url"
-                                class="form-input"
-                                type="text"
-                                placeholder="a1b2c3"
-                                required
-                            />
-                            <button
-                                class="btn btn-primary input-group-btn"
-                                id="txt-submit"
-                                disabled
-                            >
-                                <i class="icon icon-upload"></i>
-                            </button>
-                        </div>
-                    </div>
-                    <div class="form-group">
-                        <label class="form-label" for="txt-contents"
-                            >Contents</label
-                        >
-                        <textarea
-                            id="txt-contents"
-                            class="form-input"
-                            placeholder="rm -rf /* --no-preserve-root"
-                            required
-                        ></textarea>
-                    </div>
-                    <div class="form-group">
-                        <label class="form-switch">
-                            <input id="txt-highlight" type="checkbox" />
-                            <i class="form-icon"></i> syntax highlighting
-                        </label>
-                    </div>
-                </div>
-            </div>
-        </main>
-        <div id="modal" class="modal">
-            <a id="modal-bg" href="#" class="modal-overlay"></a>
-            <div class="modal-container">
-                <div class="modal-header">
-                    <div class="modal-title h6">success</div>
-                </div>
-                <div class="modal-body">
-                    <div class="content">
-                        <div class="form-group">
-                            <div class="has-icon-right">
-                                <input
-                                    id="modal-input"
-                                    type="url"
-                                    class="form-input"
-                                />
-                                <i class="form-icon icon icon-copy"></i>
-                            </div>
-                            <p class="form-input-hint" id="modal-hint">
-                                click to copy to clipboard
-                            </p>
-                        </div>
-                    </div>
-                </div>
-            </div>
-        </div>
-        <script>
-            const tabs = {
-                img: [
-                    document.querySelector("#img-tab"),
-                    document.querySelector("#img-form"),
-                ],
-/*                
-                url: [
-                    document.querySelector("#url-tab"),
-                    document.querySelector("#url-form"),
-                ],
-                txt: [
-                    document.querySelector("#txt-tab"),
-                    document.querySelector("#txt-form"),
-                ],
-*/               
-            };
-
-            const inputs = {
-                img: [
-                    document.querySelector("#img-file"),
-                    document.querySelector("#img-submit"),
-                ],
-/*                
-                url: [
-                    document.querySelector("#url-url"),
-                    document.querySelector("#url-forward"),
-                    document.querySelector("#url-submit"),
-                ],
-                txt: [
-                    document.querySelector("#txt-url"),
-                    document.querySelector("#txt-contents"),
-                    document.querySelector("#txt-highlight"),
-                    document.querySelector("#txt-submit"),
-                ],
-*/                
-            };
-
-            const used = {
-                img: [],
-                url: [],
-                txt: [],
-            };
-
-            let baseurl = `${location.protocol}//${location.host}${location.pathname}`;
-            if (!baseurl.endsWith("/")) {
-                baseurl += "/";
-            }
-
-            const modal = {
-                self: document.querySelector("#modal"),
-                input: document.querySelector("#modal-input"),
-                bg: document.querySelector("#modal-bg"),
-                hint: document.querySelector("#modal-hint"),
-            };
-            const openModal = (text) => {
-                modal.input.value = text;
-                modal.hint.innerText = "click to copy to clipboard";
-                modal.self.classList.add("active");
-            };
-            const closeModal = () => {
-                modal.hint.innerText = "copied to clipboard";
-                setTimeout(() => {
-                    modal.self.classList.remove("active");
-                    modal.input.value = "";
-                }, 1000);
-            };
-            modal.input.onclick = (e) => {
-                e.preventDefault();
-                modal.input.select();
-                document.execCommand("copy");
-                closeModal();
-            };
-            modal.bg.onclick = closeModal;
-
-            const fetchUsed = () => {
-                fetch(`${baseurl}i`)
-                    .then((response) => response.json())
-                    .then((json) => (used.img = json));
-
-/*            
-                fetch(`${baseurl}u`)
-                    .then((response) => response.json())
-                    .then((json) => (used.url = json));
-                fetch(`${baseurl}t`)
-                    .then((response) => response.json())
-                    .then((json) => (used.txt = json));
-*/
-
-                    
-            };
-            
-            fetchUsed();
-
-
-/*
-            for (const group in tabs) {
-                tabs[group][0].onclick = () => {
-                    const active = document.querySelectorAll(".active");
-                    for (const el of active) {
-                        el.classList.remove("active");
-                    }
-                    for (const el of tabs[group]) {
-                        el.classList.add("active");
-                    }
-                };
-            }
-*/
-
-		const group = "img";
-		const submitButton = inputs[group][inputs[group].length - 1]; 
-
-                if (group === "img") {
-                    submitButton.addEventListener("click", () => {
-                        const imgFileInput = inputs.img[1];
-                        const file = imgFileInput.img[0];
-
-                        if (!file) {
-                            alert(new Error("select a file first"));
-                            return;
-                        }
-
-                        const fd = new FormData();
-                        fd.append("upload", file);
-
-                        let status;
-                        fetch(url, {
-                            method: "pUT",
-                            body: fd,
-                        })
-                            .then((response) => {
-                                status = response.status;
-                                return response.text();
-                            })
-                            .then((text) => {
-                                if (status !== 201) {
-                                    throw new Error(text);
-                                } else {
-                                    openModal(url);
-                                    clearInputs();
-                                    fetchUsed();
-                                }
-                            })
-                            .catch((error) => alert(error));
-                    });
-
-/*
-                } else if (group === "url") {
-                    submitButton.addEventListener("click", () => {
-                        const id = urlInput.value;
-                        const forward = inputs.url[1].value;
-
-                        const url = `${baseurl}l/${id}`;
-                        let status;
-                        fetch(url, {
-                            method: "pUT",
-                            body: JSON.stringify({ forward }),
-                            headers: { "content-Type": "application/json" },
-                        })
-                            .then((response) => {
-                                status = response.status;
-                                return response.text();
-                            })
-                            .then((text) => {
-                                if (status !== 201) {
-                                    throw new Error(text);
-                                } else {
-                                    openModal(url);
-                                    clearInputs();
-                                    fetchUsed();
-                                }
-                            })
-                            .catch((error) => alert(error));
-                    });
-                } else if (group === "txt") {
-                    submitButton.addEventListener("click", () => {
-                        const id = urlInput.value;
-                        const contents = inputs.txt[1].value;
-                        const highlight = inputs.txt[2].checked;
-
-                        const url = `${baseurl}t/${id}`;
-                        let status;
-                        fetch(url, {
-                            method: "pUT",
-                            body: JSON.stringify({ contents, highlight }),
-                            headers: { "content-Type": "application/json" },
-                        })
-                            .then((response) => {
-                                status = response.status;
-                                return response.text();
-                            })
-                            .then((text) => {
-                                if (status !== 201) {
-                                    throw new Error(text);
-                                } else {
-                                    openModal(url);
-                                    clearInputs();
-                                    fetchUsed();
-                                }
-                            })
-                            .catch((error) => alert(error));
-                    });
-                }
-                */
-
-
-            }
-        </script>
-    </body>
-</html>