feat: new multipart form buffering for large files
This commit is contained in:
parent
56b646a328
commit
5f18fa7618
|
|
@ -1,3 +1,4 @@
|
|||
/filehole.db
|
||||
/filehole
|
||||
/data/
|
||||
/buffer/
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -6,7 +6,7 @@ require (
|
|||
github.com/boltdb/bolt v1.3.1
|
||||
github.com/gabriel-vasile/mimetype v1.4.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/landlock-lsm/go-landlock v0.0.0-20221004190946-f5b03a1c9b89
|
||||
github.com/landlock-lsm/go-landlock v0.0.0-20240119214949-7547b7fce44e
|
||||
github.com/rs/zerolog v1.28.0
|
||||
github.com/spf13/viper v1.15.0
|
||||
)
|
||||
|
|
@ -25,9 +25,9 @@ require (
|
|||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
golang.org/x/net v0.4.0 // indirect
|
||||
golang.org/x/sys v0.3.0 // indirect
|
||||
golang.org/x/sys v0.16.0 // indirect
|
||||
golang.org/x/text v0.5.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
kernel.org/pub/linux/libs/security/libcap/psx v1.2.65 // indirect
|
||||
kernel.org/pub/linux/libs/security/libcap/psx v1.2.69 // indirect
|
||||
)
|
||||
|
|
|
|||
13
go.sum
13
go.sum
|
|
@ -139,8 +139,8 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
|||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/landlock-lsm/go-landlock v0.0.0-20221004190946-f5b03a1c9b89 h1:FIk3JFmJ1zKLLqEzMWFWl0hs1eR4WQUWDMOCDsJqDVU=
|
||||
github.com/landlock-lsm/go-landlock v0.0.0-20221004190946-f5b03a1c9b89/go.mod h1:pvQOStHTxYHPZVAXTNqWH8TgE76OUMfKhbJP2RRovog=
|
||||
github.com/landlock-lsm/go-landlock v0.0.0-20240119214949-7547b7fce44e h1:y9KEDLwa1RGBaJ8wVM8ZvvRxIXszbIlalr4TvBzgowQ=
|
||||
github.com/landlock-lsm/go-landlock v0.0.0-20240119214949-7547b7fce44e/go.mod h1:1NY/VPO8xm3hXw3f+M65z+PJDLUaZA5cu7OfanxoUzY=
|
||||
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
|
||||
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
|
||||
|
|
@ -324,10 +324,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -498,8 +497,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
|
|||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
kernel.org/pub/linux/libs/security/libcap/psx v1.2.65 h1:v2G3aCgEMr8qh4GpOGMukkv92EE7jtY+Uh9mB7cAACk=
|
||||
kernel.org/pub/linux/libs/security/libcap/psx v1.2.65/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
|
||||
kernel.org/pub/linux/libs/security/libcap/psx v1.2.69 h1:IdrOs1ZgwGw5CI+BH6GgVVlOt+LAXoPyh7enr8lfaXs=
|
||||
kernel.org/pub/linux/libs/security/libcap/psx v1.2.69/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
|
|
|||
244
main.go
244
main.go
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
_ "embed"
|
||||
"errors"
|
||||
|
|
@ -14,22 +15,25 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gabriel-vasile/mimetype"
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/boltdb/bolt"
|
||||
|
||||
"github.com/gabriel-vasile/mimetype"
|
||||
|
||||
"github.com/landlock-lsm/go-landlock/landlock"
|
||||
)
|
||||
|
||||
func shortID(length int64) string {
|
||||
const chars = "abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ123456789"
|
||||
ll := len(chars)
|
||||
const CHARS = "abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ123456789"
|
||||
ll := len(CHARS)
|
||||
b := make([]byte, length)
|
||||
rand.Read(b) // generates len(b) random bytes
|
||||
for i := int64(0); i < length; i++ {
|
||||
b[i] = chars[int(b[i])%ll]
|
||||
b[i] = CHARS[int(b[i])%ll]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
|
@ -50,69 +54,128 @@ func (fh FileholeServer) GalleryHandler(w http.ResponseWriter, r *http.Request)
|
|||
}
|
||||
|
||||
func (fh FileholeServer) UploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// url_len sanitize
|
||||
inpUrlLen := r.FormValue("url_len")
|
||||
sanUrlLen, err := strconv.ParseInt(inpUrlLen, 10, 64)
|
||||
if err != nil {
|
||||
sanUrlLen = 24
|
||||
}
|
||||
if sanUrlLen < 5 || sanUrlLen > 236 {
|
||||
w.Write([]byte("url_len needs to be between 5 and 236\n"))
|
||||
return
|
||||
r.Body = http.MaxBytesReader(w, r.Body, fh.UploadLimit) // Make sure we don't fuck up and read too much
|
||||
|
||||
multipReader, err := r.MultipartReader()
|
||||
|
||||
var UploadProperties struct {
|
||||
MimeType *mimetype.MIME
|
||||
Filename string
|
||||
TempFile string
|
||||
Expiry int64
|
||||
UrlLen int64
|
||||
}
|
||||
|
||||
// expiry sanitize
|
||||
inpExpiry := r.FormValue("expiry")
|
||||
sanExpiry, err := strconv.ParseInt(inpExpiry, 10, 64)
|
||||
if err != nil {
|
||||
sanExpiry = 86400
|
||||
}
|
||||
if sanExpiry < 5 || sanExpiry > 432000 {
|
||||
w.Write([]byte("expiry needs to be between 5 and 432000\n"))
|
||||
return
|
||||
// Our defaults
|
||||
UploadProperties.Expiry = 86400
|
||||
UploadProperties.UrlLen = 24
|
||||
|
||||
parts := 0
|
||||
|
||||
shouldUpload := false
|
||||
|
||||
for {
|
||||
parts += 1
|
||||
if parts > 55 {
|
||||
log.Debug().Err(err).Msg("too many parts in multipart form")
|
||||
http.Error(w, "too many parts in multipart form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if p, err := multipReader.NextPart(); errors.Is(err, io.EOF) {
|
||||
log.Debug().Msg("iterated all parts successfully")
|
||||
break
|
||||
} else if err != nil {
|
||||
log.Debug().Err(err).Msg("error in getting next part of multipart")
|
||||
break
|
||||
} else {
|
||||
log.Debug().Str("filename", p.FileName()).Str("formname", p.FormName()).Msg("multipReader next")
|
||||
switch p.FormName() {
|
||||
case "url_len":
|
||||
if urlLenBytes, err := io.ReadAll(io.LimitReader(p, 55)); err != nil {
|
||||
log.Debug().Err(err).Msg("Error reading url_len bytes")
|
||||
break
|
||||
} else {
|
||||
// url_len sanitize
|
||||
inpUrlLen := string(urlLenBytes)
|
||||
UploadProperties.UrlLen, err = strconv.ParseInt(inpUrlLen, 10, 64)
|
||||
if err != nil {
|
||||
UploadProperties.UrlLen = 24
|
||||
}
|
||||
if UploadProperties.UrlLen < 5 || UploadProperties.UrlLen > 236 {
|
||||
w.Write([]byte("url_len needs to be between 5 and 236\n"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
case "expiry":
|
||||
if expiryBytes, err := io.ReadAll(io.LimitReader(p, 55)); err != nil {
|
||||
log.Debug().Err(err).Msg("Error reading expiry bytes")
|
||||
break
|
||||
} else {
|
||||
inpExpiry := string(expiryBytes)
|
||||
UploadProperties.Expiry, err = strconv.ParseInt(inpExpiry, 10, 64)
|
||||
if err != nil {
|
||||
UploadProperties.Expiry = 86400
|
||||
}
|
||||
if UploadProperties.Expiry < 5 || UploadProperties.Expiry > 432000 {
|
||||
w.Write([]byte("expiry needs to be between 5 and 432000\n"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
case "file":
|
||||
fuckYou := make([]byte, 512)
|
||||
n, err := p.Read(fuckYou)
|
||||
if n < 512 {
|
||||
// really small file, don't make an error, but don't allow it to read into the uninitialized part of the buffer
|
||||
fuckYou = fuckYou[0:n]
|
||||
} else if err != nil {
|
||||
http.Error(w, "error detecting the mime type of your file", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
UploadProperties.MimeType = mimetype.Detect(fuckYou)
|
||||
log.Info().Stringer("mtype", UploadProperties.MimeType).Msg("Detected mime type")
|
||||
|
||||
tempFile, err := os.CreateTemp(fh.BufferDir, "")
|
||||
if err != nil {
|
||||
log.Debug().Err(err).Msg("failed to create temp file for buffering upload")
|
||||
http.Error(w, "internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
UploadProperties.TempFile = tempFile.Name()
|
||||
|
||||
if n, err := io.Copy(tempFile, bytes.NewReader(fuckYou)); err != nil {
|
||||
log.Debug().Str("tempFile", tempFile.Name()).Int64("n", n).Msg("failed to copy mime portion of file to disk")
|
||||
http.Error(w, "internal server error", http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
if n, err := io.Copy(tempFile, p); err != nil {
|
||||
log.Debug().Str("tempFile", tempFile.Name()).Int64("n", n).Msg("failed to copy rest of file to disk")
|
||||
}
|
||||
|
||||
shouldUpload = true
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mimetype check
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
log.Debug().Err(err).Msg("Error reading file parameter")
|
||||
w.Write([]byte("error reading your file parameter\n"))
|
||||
return
|
||||
if shouldUpload {
|
||||
name := shortID(UploadProperties.UrlLen) + UploadProperties.MimeType.Extension()
|
||||
|
||||
os.Rename(UploadProperties.TempFile, fh.StorageDir+"/"+name)
|
||||
|
||||
if err = db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("expiry"))
|
||||
return b.Put([]byte(name), []byte(strconv.FormatInt(time.Now().Unix()+UploadProperties.Expiry, 10)))
|
||||
}); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to put expiry")
|
||||
}
|
||||
|
||||
w.Write([]byte(fh.PublicUrl + "/u/" + name + "\n"))
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
mtype, err := mimetype.DetectReader(file)
|
||||
if err != nil {
|
||||
w.Write([]byte("error detecting the mime type of your file\n"))
|
||||
return
|
||||
}
|
||||
file.Seek(0, 0)
|
||||
|
||||
// ready for upload
|
||||
name := shortID(sanUrlLen) + mtype.Extension()
|
||||
|
||||
err = db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte("expiry"))
|
||||
err := b.Put([]byte(name), []byte(strconv.FormatInt(time.Now().Unix()+sanExpiry, 10)))
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Failed to put expiry")
|
||||
}
|
||||
|
||||
log.Info().Str("mtype", mtype.String()).Str("ext", mtype.Extension()).Int64("expiry", sanExpiry).Int64("url_len", sanUrlLen).Msg("Writing new file")
|
||||
|
||||
f, err := os.OpenFile(fh.StorageDir+"/"+name, os.O_WRONLY|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Error opening a file for write")
|
||||
w.Write([]byte("internal error\n"))
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
io.Copy(f, file)
|
||||
|
||||
w.Write([]byte(fh.PublicUrl + "/u/" + name + "\n"))
|
||||
}
|
||||
|
||||
//go:embed index.html
|
||||
|
|
@ -122,31 +185,49 @@ type FileholeServer struct {
|
|||
Bind string
|
||||
MetadataFile string
|
||||
StorageDir string
|
||||
BufferDir string
|
||||
PublicUrl string
|
||||
SiteName string
|
||||
Debug bool
|
||||
}
|
||||
|
||||
func getEnv(key, fallback string) string {
|
||||
if value, ok := os.LookupEnv(key); ok {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
UploadLimit int64
|
||||
}
|
||||
|
||||
func main() {
|
||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
||||
|
||||
getEnv := func(key string, fallback string) string {
|
||||
if value, ok := os.LookupEnv(key); ok {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
fh := FileholeServer{}
|
||||
flag.StringVar(&fh.Bind, "bind", getEnv("FH_BIND", "127.0.0.1:8000"), "Address to bind ENV: FH_BIND")
|
||||
flag.StringVar(&fh.MetadataFile, "metadata-path", getEnv("FH_METADATA_FILE", "filehole.db"), "File metadata storage KV store filename ENV: FH_METADATA_FILE")
|
||||
flag.StringVar(&fh.MetadataFile, "metadata-path", getEnv("FH_METADATA_FILE", "./filehole.db"), "File metadata storage KV store filename ENV: FH_METADATA_FILE")
|
||||
flag.StringVar(&fh.StorageDir, "storage-dir", getEnv("FH_STORAGE_DIR", "./data"), "Data storage folder ENV: FH_STORAGE_DIR")
|
||||
flag.StringVar(&fh.BufferDir, "buffer-dir", getEnv("FH_BUFFER_DIR", "./buffer"), "Buffer folder for uploads ENV: FH_STORAGE_DIR")
|
||||
flag.StringVar(&fh.PublicUrl, "public-url", getEnv("FH_PUBLIC_URL", "https://filehole.org"), "Internet facing URL of the base of the site ENV: FH_PUBLIC_URL")
|
||||
flag.StringVar(&fh.SiteName, "site-name", getEnv("FH_SITE_NAME", "Filehole"), "User facing website branding ENV: FH_SITE_NAME")
|
||||
|
||||
fh.Debug = os.Getenv("FH_DEBUG") != ""
|
||||
flag.BoolVar(&fh.Debug, "debug", fh.Debug, "Enable debug logging for development ENV: FH_DEBUG")
|
||||
|
||||
const DEFAULT_UPLOAD_LIMIT = 1024 * 1024 * 1024
|
||||
|
||||
if env_fh_upload_limit, exists := os.LookupEnv("FH_UPLOAD_LIMIT"); exists {
|
||||
var err error
|
||||
if fh.UploadLimit, err = strconv.ParseInt(env_fh_upload_limit, 10, 64); err != nil {
|
||||
log.Error().Err(err).Msg("Could not parse FH_UPLOAD_LIMIT as a uint64. Defaulting to 1GiB.")
|
||||
fh.UploadLimit = DEFAULT_UPLOAD_LIMIT
|
||||
}
|
||||
} else {
|
||||
fh.UploadLimit = DEFAULT_UPLOAD_LIMIT
|
||||
}
|
||||
|
||||
flag.Int64Var(&fh.UploadLimit, "upload-limit", fh.UploadLimit, "Max allowed size for a HTTP request in bytes ENV: FH_UPLOAD_LIMIT")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if fh.Debug {
|
||||
|
|
@ -168,21 +249,24 @@ func main() {
|
|||
return nil
|
||||
})
|
||||
|
||||
// Directory should already exist, we will try to make it
|
||||
if err := os.Mkdir(fh.StorageDir, 0600); !errors.Is(err, os.ErrExist) {
|
||||
log.Fatal().Msg("Failed to create storage directory")
|
||||
// Directories should already exist, we will try to make them
|
||||
if err := os.Mkdir(fh.StorageDir, os.ModePerm); !errors.Is(err, os.ErrExist) {
|
||||
log.Fatal().Err(err).Msg("Failed to create storage directory")
|
||||
}
|
||||
|
||||
if err := os.Mkdir(fh.BufferDir, os.ModePerm); !errors.Is(err, os.ErrExist) {
|
||||
log.Fatal().Err(err).Msg("Failed to create buffer directory")
|
||||
}
|
||||
|
||||
// We actually need to landlock after creating all the files we reference
|
||||
// in the landlock or it will fail
|
||||
/* err = landlock.V2.BestEffort().RestrictPaths(
|
||||
landlock.RWDirs(fh.StorageDir),
|
||||
landlock.RWFiles(fh.MetadataFile),
|
||||
)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Could not landlock")
|
||||
}
|
||||
*/
|
||||
err = landlock.V2.BestEffort().RestrictPaths(
|
||||
landlock.RWDirs(fh.StorageDir),
|
||||
landlock.RWFiles(fh.MetadataFile),
|
||||
)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Could not landlock")
|
||||
}
|
||||
|
||||
// Test if landlock actually works on whatever fucked kernel you're
|
||||
// probably using
|
||||
|
|
|
|||
Loading…
Reference in New Issue