From fa0b2f104eafe11a168e745e7fbff85398ab7b92 Mon Sep 17 00:00:00 2001 From: "kayos@tcp.direct" Date: Sat, 12 Mar 2022 00:43:05 -0800 Subject: [PATCH] Feat: Working PoC --- cmd/keepr/getfiles.go | 28 +++++--- internal/collect/collection.go | 118 +++++++++++++++++++++++++++------ internal/collect/ingest.go | 22 ++++++ internal/collect/parse.go | 85 ++++++++++++++++-------- internal/config/init.go | 14 +++- 5 files changed, 205 insertions(+), 62 deletions(-) diff --git a/cmd/keepr/getfiles.go b/cmd/keepr/getfiles.go index acc073d..c6ff472 100644 --- a/cmd/keepr/getfiles.go +++ b/cmd/keepr/getfiles.go @@ -4,6 +4,8 @@ import ( "os" "runtime" "strings" + "sync/atomic" + "time" "github.com/rs/zerolog" "kr.dev/walk" @@ -44,34 +46,38 @@ func main() { slog.Debug().Msg("skiping directory entirely") cripwalk.SkipDir() } - slog.Trace().Msg("directory") + // slog.Trace().Msg("directory") case cripwalk.Path() == os.Args[1]: slog.Debug().Msg("skipping self-parent directory entirely") cripwalk.SkipParent() default: sample, err := collect.Process(cripwalk.Entry(), util.APath(cripwalk.Path(), config.Relative)) if err != nil { - log.Warn().Err(err).Msgf("failed to process %s", cripwalk.Entry().Name()) + slog.Warn().Err(err).Msgf("failed to process") continue } if sample == nil { - log.Trace().Msgf("skipping unknown file %s", cripwalk.Entry().Name()) + slog.Trace().Msgf("skipping unknown file") continue } - collect.Library.IngestTempo(sample) } } - if zerolog.GlobalLevel() == zerolog.TraceLevel { - collect.Library.TempoStats() + for !atomic.CompareAndSwapInt32(&collect.Backlog, 0, -1) { + time.Sleep(1 * time.Second) + print(".") } if config.StatsOnly { - return + collect.Library.TempoStats() + collect.Library.KeyStats() + collect.Library.DrumStats() } - err := collect.Library.SymlinkTempos() - if err != nil { - log.Fatal().Err(err).Msg("returned from symlinkTempos") - } + var errs []error + errs = append(errs, collect.Library.SymlinkTempos()) + errs = append(errs, collect.Library.SymlinkKeys()) + errs = append(errs, collect.Library.SymlinkDrums()) + + log.Info().Errs("errs", errs).Msg("fin.") } diff --git a/internal/collect/collection.go b/internal/collect/collection.go index 4aeff8d..48be017 100644 --- a/internal/collect/collection.go +++ b/internal/collect/collection.go @@ -97,22 +97,62 @@ func (c *Collection) TempoStats() { } } +// DrumStats outputs the amount of samples of each known drum type. +func (c *Collection) DrumStats() { + c.mu.RLock() + defer c.mu.RUnlock() + for t, ss := range c.Drums { + if len(ss) > 1 { + log.Printf("%s: %d", drumToDirMap[t], len(ss)) + } + } +} + +// KeyStats outputs the amount of samples with each known key. +func (c *Collection) KeyStats() { + c.mu.RLock() + defer c.mu.RUnlock() + for t, ss := range c.Keys { + if len(ss) > 1 { + log.Printf("%s: %d", t.Root.String(t.AdjSymbol), len(ss)) + } + } +} + +func link(sample *Sample, kp string) { + slog := log.With().Str("caller", sample.Path).Logger() + finalPath := kp + sample.Name + slog.Trace().Msg(finalPath) + err := freshLink(finalPath) + if err != nil { + slog.Warn().Err(err).Msg("old symlink delete failure") + } + if _, err = os.Stat(sample.Path); err != nil { + slog.Warn().Err(err).Msg("can't stat original file") + } + if config.Simulate { + log.Printf("would have linked %s -> %s", sample.Path, finalPath) + return + } + err = os.Symlink(sample.Path, finalPath) + if err != nil && !os.IsNotExist(err) { + slog.Error().Err(err).Msg("failed to create symlink") + } +} + func (c *Collection) SymlinkTempos() (err error) { log.Trace().Msg("SymlinkTempos start") defer log.Trace().Err(err).Msg("SymlinkTempos finish") c.mu.RLock() defer c.mu.RUnlock() - if len(c.Tempos) < 1 { - return errors.New("no tempos recorded") + return errors.New("no known tempos") } - dst := util.APath(config.Destination+"Tempo", config.Relative) err = os.MkdirAll(dst, os.ModePerm) if err != nil && !os.IsNotExist(err) { return } - for t, ss := range c.Tempos { tempopath := dst + "/" + strconv.Itoa(t) + "/" err = os.MkdirAll(tempopath, os.ModePerm) @@ -120,21 +160,61 @@ func (c *Collection) SymlinkTempos() (err error) { return } for _, s := range ss { - go func(sample *Sample) { - finalPath := tempopath + sample.Name - log.Trace().Str("caller", sample.Path).Msg(finalPath) - err = freshLink(finalPath) - if err != nil { - return - } - if _, err = os.Stat(sample.Path); err != nil { - return - } - err = os.Symlink(sample.Path, finalPath) - if err != nil && !os.IsNotExist(err) { - log.Error().Err(err).Msg("failed to create symlink") - } - }(s) + go link(s, tempopath) + } + } + return nil +} + +func (c *Collection) SymlinkKeys() (err error) { + log.Trace().Msg("SymlinkKeys start") + defer log.Trace().Err(err).Msg("SymlinkKeys finish") + c.mu.RLock() + defer c.mu.RUnlock() + + if len(c.Keys) < 1 { + return errors.New("no known keys") + } + dst := util.APath(config.Destination+"Key", config.Relative) + err = os.MkdirAll(dst, os.ModePerm) + if err != nil && !os.IsNotExist(err) { + return + } + for t, ss := range c.Keys { + keypath := dst + "/" + t.Root.String(t.AdjSymbol) + "/" + err = os.MkdirAll(keypath, os.ModePerm) + if err != nil && !os.IsExist(err) { + return + } + for _, s := range ss { + go link(s, keypath) + } + } + return nil +} + +func (c *Collection) SymlinkDrums() (err error) { + log.Trace().Msg("SymlinkDrums start") + defer log.Trace().Err(err).Msg("SymlinkDrums finish") + c.mu.RLock() + defer c.mu.RUnlock() + + if len(c.Drums) < 1 { + return errors.New("no known drums") + } + dst := util.APath(config.Destination+"Drums", config.Relative) + err = os.MkdirAll(dst, os.ModePerm) + if err != nil && !os.IsNotExist(err) { + return + } + for t, ss := range c.Drums { + drumpath := dst + "/" + drumToDirMap[t] + "/" + err = os.MkdirAll(drumpath, os.ModePerm) + if err != nil && !os.IsExist(err) { + return + } + for _, s := range ss { + go link(s, drumpath) } } return nil diff --git a/internal/collect/ingest.go b/internal/collect/ingest.go index e00e126..dbfa45f 100644 --- a/internal/collect/ingest.go +++ b/internal/collect/ingest.go @@ -1,7 +1,14 @@ package collect +import "sync/atomic" + +var Backlog int32 + // IngestKey creates a map of tempo to sample. func (c *Collection) IngestKey(sample *Sample) { + log.Debug().Str("caller", sample.Name).Msgf("Key: %s", sample.Key.Root.String(sample.Key.AdjSymbol)) + atomic.AddInt32(&Backlog, 1) + defer atomic.AddInt32(&Backlog, -1) c.mu.Lock() defer c.mu.Unlock() c.Keys[sample.Key] = append(c.Keys[sample.Key], sample) @@ -9,6 +16,12 @@ func (c *Collection) IngestKey(sample *Sample) { // IngestTempo creates a map of tempo to sample. func (c *Collection) IngestTempo(sample *Sample) { + if sample.Tempo == 0 || sample.Tempo < 50 || sample.Tempo > 250 { + return + } + log.Debug().Str("caller", sample.Name).Msgf("Tempo: %d", sample.Tempo) + atomic.AddInt32(&Backlog, 1) + defer atomic.AddInt32(&Backlog, -1) c.mu.Lock() defer c.mu.Unlock() c.Tempos[sample.Tempo] = append(c.Tempos[sample.Tempo], sample) @@ -16,6 +29,9 @@ func (c *Collection) IngestTempo(sample *Sample) { // IngestMelodicLoop appends to a list of [pointers to] melodic loop samples. func (c *Collection) IngestMelodicLoop(sample *Sample) { + log.Debug().Str("caller", sample.Name).Msg("Melodic Loop") + atomic.AddInt32(&Backlog, 1) + defer atomic.AddInt32(&Backlog, -1) c.mu.Lock() defer c.mu.Unlock() c.MelodicLoops = append(c.MelodicLoops, sample) @@ -23,6 +39,9 @@ func (c *Collection) IngestMelodicLoop(sample *Sample) { // IngestMIDI appends to a list of [pointers to] MIDI/SMF files. func (c *Collection) IngestMIDI(sample *Sample) { + log.Debug().Str("caller", sample.Name).Msg("MIDI") + atomic.AddInt32(&Backlog, 1) + defer atomic.AddInt32(&Backlog, -1) c.mu.Lock() defer c.mu.Unlock() c.Midis = append(c.Midis, sample) @@ -30,6 +49,9 @@ func (c *Collection) IngestMIDI(sample *Sample) { // IngestDrum creates a map of different drum types to samples. func (c *Collection) IngestDrum(sample *Sample, drumType DrumType) { + log.Debug().Str("caller", sample.Name).Msgf("Drum: %s", drumToDirMap[drumType]) + atomic.AddInt32(&Backlog, 1) + defer atomic.AddInt32(&Backlog, -1) c.mu.Lock() defer c.mu.Unlock() c.Drums[drumType] = append(c.Drums[drumType], sample) diff --git a/internal/collect/parse.go b/internal/collect/parse.go index b35b78d..fd3d407 100644 --- a/internal/collect/parse.go +++ b/internal/collect/parse.go @@ -6,10 +6,12 @@ import ( "os" "strconv" "strings" + "sync/atomic" "github.com/go-audio/wav" "gopkg.in/music-theory.v0/key" - "gopkg.in/music-theory.v0/note" + + "git.tcp.direct/kayos/keepr/internal/config" ) func freshLink(path string) error { @@ -21,7 +23,11 @@ func freshLink(path string) error { return nil } -func checkbpm(piece string) (bpm int) { +func guessBPM(piece string) (bpm int) { + // TODO: don't trust this lol? + if num, numerr := strconv.Atoi(piece); numerr != nil { + return num + } frg := strings.Split(piece, "bpm")[0] m := strings.Split(frg, "") var start = 0 @@ -76,31 +82,45 @@ var drumDirMap = map[string]DrumType{ "open_hihats": HatOpen, "808s": EightOhEight, "808": EightOhEight, "toms": Tom, } +var drumToDirMap = map[DrumType]string{ + Snare: "Snares", Kick: "Kicks", HiHat: "HiHats", HatClosed: "HiHat/Closed", + HatOpen: "HiHat/Open", EightOhEight: "808", Tom: "Toms", Percussion: "Other", +} + func (s *Sample) ParseFilename() { - for _, piece := range guessSeperator(s.Name) { - piece = strings.ToLower(piece) + atomic.AddInt32(&Backlog, 1) + defer atomic.AddInt32(&Backlog, -1) + drumtype, isdrum := drumDirMap[s.getParentDir()] + + switch { + case s.getParentDir() == "melodic_loops": + if !s.IsType(Loop) { + s.Type = append(s.Type, Loop) + go Library.IngestMelodicLoop(s) + } + case isdrum: + go Library.IngestDrum(s, drumtype) + } + + for _, opiece := range guessSeperator(s.Name) { + piece := strings.ToLower(opiece) + if num, numerr := strconv.Atoi(piece); numerr == nil { + if num > 50 && num != 808 { + s.Tempo = num + } + } if strings.Contains(piece, "bpm") { - s.Tempo = checkbpm(piece) - if s.Tempo != 0 { - s.Type = append(s.Type, Loop) - } + s.Tempo = guessBPM(piece) } - drumtype, isdrum := drumDirMap[s.getParentDir()] - - switch { - case s.getParentDir() == "melodic_loops": - if !s.IsType(Loop) { - s.Type = append(s.Type, Loop) - go Library.IngestMelodicLoop(s) - } - if isdrum { - go Library.IngestDrum(s, drumtype) - } + if s.Tempo != 0 { + go Library.IngestTempo(s) } - spl := strings.Split(piece, "") - + spl := strings.Split(opiece, "") + if len(spl) < 1 { + continue + } // if our fragment starts with a known root note, then try to parse the fragment, else dip-set. switch spl[0] { case "C", "D", "E", "F", "G", "A", "B": @@ -109,11 +129,8 @@ func (s *Sample) ParseFilename() { continue } - k := key.Of(piece) - if k.Root != note.Nil { - s.Key = k - Library.IngestKey(s) - } + s.Key = key.Of(opiece) + go Library.IngestKey(s) } } @@ -133,6 +150,10 @@ func readWAV(s *Sample) error { if decoder.Err() != nil { return decoder.Err() } + if meta := decoder.Metadata; meta == nil { + return nil + } + s.Metadata = decoder.Metadata log.Trace().Msg(fmt.Sprintf("metadata: %v", s.Metadata)) @@ -142,6 +163,7 @@ func readWAV(s *Sample) error { } func Process(entry fs.DirEntry, dir string) (s *Sample, err error) { + log.Trace().Str("caller", entry.Name()).Msg("Processing") var finfo os.FileInfo finfo, err = entry.Info() if err != nil { @@ -159,13 +181,18 @@ func Process(entry fs.DirEntry, dir string) (s *Sample, err error) { switch ext { case "midi", "mid": - s.Type = append(s.Type, MIDI) + if !config.NoMIDI { + s.Type = append(s.Type, MIDI) + go Library.IngestMIDI(s) + } case "wav": - err = readWAV(s) + if !config.SkipWavDecode { + err = readWAV(s) + } if err != nil { return nil, err } - s.ParseFilename() + go s.ParseFilename() default: return nil, nil } diff --git a/internal/config/init.go b/internal/config/init.go index 68ac5af..882e899 100644 --- a/internal/config/init.go +++ b/internal/config/init.go @@ -16,9 +16,11 @@ var ( // Destination is the base path for our symlink library. Destination = defDestination // Relative will determine if we use relative pathing for symlinks. - Relative = false - - StatsOnly = false + Relative = false + Simulate = false + StatsOnly = false + NoMIDI = false + SkipWavDecode = false ) // GetLogger retrieves a pointer to our zerolog instance. @@ -55,6 +57,12 @@ func init() { Relative = true case "--stats", "-s": StatsOnly = true + case "--no-op", "-n": + Simulate = true + case "--no-midi", "-m": + NoMIDI = true + case "--fast", "-f": + SkipWavDecode = true default: Target = strings.Trim(arg, "/") println("search target detected: " + Target)