From 53c761a9c156d4df1c3c12fe664ed14367ae77c4 Mon Sep 17 00:00:00 2001
From: Denis Arh <denis.arh@gmail.com>
Date: Wed, 10 Feb 2021 11:47:11 +0100
Subject: [PATCH] Fix broken resource monitoring

---
 app/boot_levels.go     |  2 +
 pkg/monitor/monitor.go | 84 ++++++++++++++++++++++++------------------
 2 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/app/boot_levels.go b/app/boot_levels.go
index ca4a539d3..2929cda41 100644
--- a/app/boot_levels.go
+++ b/app/boot_levels.go
@@ -340,6 +340,8 @@ func (app *CortezaApp) Activate(ctx context.Context) (err error) {
 		fedService.Watchers(ctx)
 	}
 
+	monitor.Watcher(ctx)
+
 	rbac.Global().Watch(ctx)
 
 	if err = sysService.Activate(ctx); err != nil {
diff --git a/pkg/monitor/monitor.go b/pkg/monitor/monitor.go
index 740deb33d..9e62f3c59 100644
--- a/pkg/monitor/monitor.go
+++ b/pkg/monitor/monitor.go
@@ -32,59 +32,73 @@ var (
 )
 
 func Setup(logger *zap.Logger, o options.MonitorOpt) {
-	log = logger.Named("monitor")
+	log = logger.Named("monitor").WithOptions(zap.AddStacktrace(zap.PanicLevel))
 	opt = o
 }
 
 func Watcher(ctx context.Context) {
-	if opt.Interval > 0 {
-		go NewMonitor(int(opt.Interval / time.Second))
-		log.Debug("watcher initialized")
+	if opt.Interval == 0 {
+		return
 	}
+
+	if opt.Interval < time.Second {
+		log.Warn("monitoring interval less than 1 second, disabling")
+		return
+	}
+
+	go NewMonitor(ctx, opt.Interval)
+	log.Debug("watcher initialized", zap.Duration("interval", opt.Interval))
 }
 
-func NewMonitor(duration int) {
+func NewMonitor(ctx context.Context, interval time.Duration) {
 	var (
 		m          = Monitor{}
 		rtm        runtime.MemStats
 		goroutines = expvar.NewInt("num_goroutine")
+		ticker     = time.NewTicker(interval)
 	)
-	var interval = time.Duration(duration) * time.Second
+
+	defer ticker.Stop()
+
 	for {
-		<-time.After(interval)
+		select {
+		case <-ticker.C:
+			// Read full mem stats
+			runtime.ReadMemStats(&rtm)
 
-		// Read full mem stats
-		runtime.ReadMemStats(&rtm)
+			// Number of goroutines
+			m.NumGoroutine = runtime.NumGoroutine()
+			goroutines.Set(int64(m.NumGoroutine))
 
-		// Number of goroutines
-		m.NumGoroutine = runtime.NumGoroutine()
-		goroutines.Set(int64(m.NumGoroutine))
+			// Misc memory stats
+			m.Alloc = rtm.Alloc
+			m.TotalAlloc = rtm.TotalAlloc
+			m.Sys = rtm.Sys
+			m.Mallocs = rtm.Mallocs
+			m.Frees = rtm.Frees
 
-		// Misc memory stats
-		m.Alloc = rtm.Alloc
-		m.TotalAlloc = rtm.TotalAlloc
-		m.Sys = rtm.Sys
-		m.Mallocs = rtm.Mallocs
-		m.Frees = rtm.Frees
+			// Live objects = Mallocs - Frees
+			m.LiveObjects = m.Mallocs - m.Frees
 
-		// Live objects = Mallocs - Frees
-		m.LiveObjects = m.Mallocs - m.Frees
+			// GC Stats
+			m.PauseTotalNs = rtm.PauseTotalNs
+			m.NumGC = rtm.NumGC
 
-		// GC Stats
-		m.PauseTotalNs = rtm.PauseTotalNs
-		m.NumGC = rtm.NumGC
+			log.With(
+				zap.Uint64("alloc", m.Alloc),
+				zap.Uint64("totalAlloc", m.TotalAlloc),
+				zap.Uint64("sys", m.Sys),
+				zap.Uint64("mallocs", m.Mallocs),
+				zap.Uint64("frees", m.Frees),
+				zap.Uint64("liveObjects", m.LiveObjects),
+				zap.Uint64("pauseTotalNs", m.PauseTotalNs),
+				zap.Uint32("numGC", m.NumGC),
+				zap.Int("numGoRoutines", m.NumGoroutine),
+			).Info("tick")
+
+		case <-ctx.Done():
+			return
+		}
 
-		log.With(
-			zap.Uint64("alloc", m.Alloc),
-			zap.Uint64("totalAlloc", m.TotalAlloc),
-			zap.Uint64("sys", m.Sys),
-			zap.Uint64("mallocs", m.Mallocs),
-			zap.Uint64("frees", m.Frees),
-			zap.Uint64("liveObjects", m.LiveObjects),
-			zap.Uint64("pauseTotalNs", m.PauseTotalNs),
-			zap.Uint32("numGC", m.NumGC),
-			zap.Int("numGoRoutines", m.NumGoroutine),
-		).
-			Info("tick")
 	}
 }