diff --git a/cmd/keepr/getfiles.go b/cmd/keepr/getfiles.go index 1ad926e..b44e51e 100644 --- a/cmd/keepr/getfiles.go +++ b/cmd/keepr/getfiles.go @@ -2,6 +2,7 @@ package main import ( "os" + "path/filepath" "runtime" "strings" "sync/atomic" @@ -30,10 +31,14 @@ func init() { func main() { log = config.GetLogger() var lastpath = "" - cripwalk := walk.New(os.DirFS(basepath), config.Target) + target := strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(config.Source), "/"), "/") + cripwalk := walk.New(os.DirFS(basepath), target) + _, output := filepath.Split(strings.TrimSuffix(config.Output, "/")) + log.Trace().Msgf("output is %s", output) for cripwalk.Next() { if err := cripwalk.Err(); err != nil { - log.Warn().Str("caller", lastpath).Msg(err.Error()) + log.Fatal().Caller().Str("caller", lastpath).Msg(err.Error()) + continue } lastpath = cripwalk.Path() slog := log.With().Str("caller", cripwalk.Path()).Logger() @@ -42,21 +47,28 @@ func main() { slog.Trace().Msg("nil") continue case cripwalk.Entry().IsDir(): - if strings.Contains(cripwalk.Path(), config.Destination) { - slog.Debug().Msg("skiping directory entirely") + if strings.Contains(cripwalk.Path(), output) { + slog.Info().Msg("skiping directory entirely") cripwalk.SkipDir() } + continue // slog.Trace().Msg("directory") default: + if strings.Contains(cripwalk.Path(), config.Output) { + log.Trace().Msg("skipping file in destination") + cripwalk.SkipDir() + continue + } sample, err := collect.Process(cripwalk.Entry(), util.APath(cripwalk.Path(), config.Relative)) if err != nil { - slog.Warn().Err(err).Msgf("failed to process") + slog.Warn().Caller().Str("caller", cripwalk.Path()).Err(err).Msgf("failed to process") continue } if sample == nil { slog.Trace().Msgf("skipping unknown file") continue } + slog.Info().Interface("sample", sample).Msg("processed") } } diff --git a/go.mod b/go.mod index e747886..5a5dee4 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,19 @@ module git.tcp.direct/kayos/keepr go 1.17 require ( + git.tcp.direct/kayos/common/squish v0.0.0-20220210125455-40e3d2190a52 github.com/go-audio/wav v1.0.0 github.com/go-music-theory/music-theory v0.0.4 github.com/rs/zerolog v1.26.1 + gopkg.in/music-theory.v0 v0.0.4 kr.dev/walk v0.1.0 ) require ( - git.tcp.direct/kayos/common/squish v0.0.0-20220210125455-40e3d2190a52 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-audio/audio v1.0.0 // indirect github.com/go-audio/riff v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/music-theory.v0 v0.0.4 // indirect gopkg.in/stretchr/testify.v1 v1.2.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/internal/collect/collection.go b/internal/collect/collection.go index ea4d7d0..c26ee7d 100644 --- a/internal/collect/collection.go +++ b/internal/collect/collection.go @@ -121,6 +121,19 @@ func (c *Collection) KeyStats() { } func link(sample *Sample, kp string) { + mapMu.RLock() + if _, ok := lockMap[sample.Path]; !ok { + mapMu.RUnlock() + mapMu.Lock() + lockMap[sample.Path] = &sync.Mutex{} + mapMu.Unlock() + mapMu.RLock() + } + defer mapMu.RUnlock() + + lockMap[sample.Path].Lock() + defer lockMap[sample.Path].Unlock() + atomic.AddInt32(&Backlog, 1) defer atomic.AddInt32(&Backlog, -1) slog := log.With().Str("caller", sample.Path).Logger() @@ -153,7 +166,7 @@ func (c *Collection) SymlinkTempos() (err error) { if len(c.Tempos) < 1 { return errors.New("no known tempos") } - dst := util.APath(config.Destination+"Tempo", config.Relative) + dst := util.APath(config.Output+"Tempo", config.Relative) err = os.MkdirAll(dst, os.ModePerm) if err != nil && !os.IsNotExist(err) { return @@ -182,7 +195,7 @@ func (c *Collection) SymlinkKeys() (err error) { if len(c.Keys) < 1 { return errors.New("no known keys") } - dst := util.APath(config.Destination+"Key", config.Relative) + dst := util.APath(config.Output+"Key", config.Relative) err = os.MkdirAll(dst, os.ModePerm) if err != nil && !os.IsNotExist(err) { return @@ -211,7 +224,7 @@ func (c *Collection) SymlinkDrums() (err error) { if len(c.Drums) < 1 { return errors.New("no known drums") } - dst := util.APath(config.Destination+"Drums", config.Relative) + dst := util.APath(config.Output+"Drums", config.Relative) err = os.MkdirAll(dst, os.ModePerm) if err != nil && !os.IsNotExist(err) { return diff --git a/internal/collect/parse.go b/internal/collect/parse.go index 496e018..aeab503 100644 --- a/internal/collect/parse.go +++ b/internal/collect/parse.go @@ -7,6 +7,7 @@ import ( "regexp" "strconv" "strings" + "sync" "sync/atomic" "github.com/go-audio/wav" @@ -15,6 +16,10 @@ import ( "git.tcp.direct/kayos/keepr/internal/config" ) +var lockMap = make(map[string]*sync.Mutex) + +var mapMu = &sync.RWMutex{} + func freshLink(path string) error { if _, err := os.Lstat(path); err == nil { if err := os.Remove(path); err != nil { @@ -104,13 +109,13 @@ func (s *Sample) ParseFilename() { drumtype, isdrum := drumDirMap[s.getParentDir()] switch { - case s.getParentDir() == "melodic_loops": + case s.getParentDir() == "melodic_loops", strings.Contains(s.getParentDir(), "melod"): if !s.IsType(Loop) { s.Type = append(s.Type, Loop) - go Library.IngestMelodicLoop(s) + Library.IngestMelodicLoop(s) } case isdrum: - go Library.IngestDrum(s, drumtype) + Library.IngestDrum(s, drumtype) } for _, opiece := range guessSeperator(s.Name) { @@ -124,8 +129,12 @@ func (s *Sample) ParseFilename() { s.Tempo = guessBPM(piece) } + if strings.Contains(s.Name, "bpm") { + continue + } + if s.Tempo != 0 { - go Library.IngestTempo(s) + Library.IngestTempo(s) } spl := strings.Split(opiece, "") @@ -153,7 +162,7 @@ func (s *Sample) ParseFilename() { } s.Key = key.Of(opiece) - go Library.IngestKey(s) + Library.IngestKey(s) } } @@ -206,7 +215,7 @@ func Process(entry fs.DirEntry, dir string) (s *Sample, err error) { case "midi", "mid": if !config.NoMIDI { s.Type = append(s.Type, MIDI) - go Library.IngestMIDI(s) + Library.IngestMIDI(s) } case "wav": if !config.SkipWavDecode { @@ -215,7 +224,7 @@ func Process(entry fs.DirEntry, dir string) (s *Sample, err error) { if err != nil { return nil, err } - go s.ParseFilename() + s.ParseFilename() default: return nil, nil } diff --git a/internal/config/init.go b/internal/config/init.go index f85fc1d..4fe0f78 100644 --- a/internal/config/init.go +++ b/internal/config/init.go @@ -13,9 +13,9 @@ const defDestination = "001-LINKED_SORTED_DIRECTORIES" var ( log zerolog.Logger - Target = "" - // Destination is the base path for our symlink library. - Destination = defDestination + Source = "" + // Output is the base path for our symlink library. + Output = defDestination // Relative will determine if we use relative pathing for symlinks. Relative = false Simulate = false @@ -60,9 +60,9 @@ func init() { switch arg { case "_": continue - case "--destination", "-d": + case "-o", "--output": required(i) - Destination = os.Args[i+1] + Output = os.Args[i+1] os.Args[i+1] = "_" case "--debug", "-v": zerolog.SetGlobalLevel(zerolog.DebugLevel) @@ -70,7 +70,7 @@ func init() { zerolog.SetGlobalLevel(zerolog.TraceLevel) case "--relative", "-r": Relative = true - case "--stats", "-s": + case "--stats": StatsOnly = true case "--no-op", "-n": Simulate = true @@ -78,33 +78,37 @@ func init() { NoMIDI = true case "--fast", "-f": SkipWavDecode = true + case "--source", "-s": + required(i) + Source = os.Args[i+1] + os.Args[i+1] = "_" default: - Target = strings.Trim(arg, "/") + log.Fatal().Msg("unknown argument: " + arg) } } - if Target == "" { + if Source == "" { log.Fatal().Msg("missing target search directory") } - f, err := os.Stat(Destination) + f, err := os.Stat(Output) switch { case err != nil: if !os.IsNotExist(err) { - log.Fatal().Caller().Str("caller", Destination).Err(err).Msg("") + log.Fatal().Caller().Str("caller", Output).Err(err).Msg("") } - if err := os.MkdirAll(Destination, os.ModePerm); err != nil { - log.Fatal().Caller().Str("caller", Destination).Err(err).Msg("could not make directory") + if err := os.MkdirAll(Output, os.ModePerm); err != nil { + log.Fatal().Caller().Str("caller", Output).Err(err).Msg("could not make directory") } case !f.IsDir(): - if Destination != "./"+defDestination { - log.Fatal().Caller().Str("caller", Destination).Msg("not a directory") + if Output != "./"+defDestination { + log.Fatal().Caller().Str("caller", Output).Msg("not a directory") } - if err := os.MkdirAll(Destination, os.ModePerm); err != nil { - log.Fatal().Caller().Str("caller", Destination).Err(err).Msg("could not make directory") + if err := os.MkdirAll(Output, os.ModePerm); err != nil { + log.Fatal().Caller().Str("caller", Output).Err(err).Msg("could not make directory") } } - if !strings.HasSuffix(Destination, "/") { - Destination = Destination + "/" + if !strings.HasSuffix(Output, "/") { + Output = Output + "/" } }