(sorry for bad english)
i am working on a file watcher that will monitor the entire C:\Users tree. but its using too much RAM(around 10 GB). i think there is a memory leak.
i am using "github.com/fsnotify/fsnotify"
and path/filepath
libraries.
the watcher only watches create and remove events, i ran filepath.WalkDir()
without watcher adding the directories and its not the problem.
go func() {//handling events
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
if event.Has(fsnotify.Create) {
log.Println("file created: ", event.Name)
} else if event.Has(fsnotify.Remove) {
log.Println("file removed: ", event.Name)
}
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Println("error:", err)
}
}
}()
var info fs.FileInfo
var err1 error
err = filepath.WalkDir("C:\\Users", func(path string, di fs.DirEntry, err error) error {
info, err1 = os.Stat(path)
if err1 != nil {
log.Print("os.Stat error:")
log.Println(err)
}
if info.IsDir() {
err1 = watcher.Add(path)
if err1 != nil {
log.Println("watcher.add error: ", path)
log.Println(err1)
}
}
return nil
})
if err != nil {
log.Fatalf("error with walkdir: %v", err)
}
log.Println("DONE")
<-make(chan struct{})
There's no memory leak, it's just that you're using fsnotify
, which on Windows allocates a 64KB buffer per watched path. See: https://github.com/fsnotify/fsnotify/blob/v1.7.0/backend_windows.go#L274
You're watching 163902 paths (according to your comments), and 163902 * 64KB is very close to 10GB.
You can configure a smaller buffer size (the minimum is 4KB) using watcher.AddWith(path, fsnotify.WithBufferSize(4096))
instead of watcher.Add(path)
. The documentation suggests that this will increase the chance of an event overflow (fsnotify.ErrEventOverflow
) if there's a "large burst of events", which may or may not be a problem for your use-case.