Rework/cleanup log stacktrace level, add LOG_STACKTRACE_LEVEL to control it

app/boot_levels.go

@@ -61,7 +61,7 @@ func (app *CortezaApp) Setup() (err error) {
 
 	{
 		// Raise warnings about experimental parts that are enabled
-		log := app.Log.WithOptions(zap.AddStacktrace(zap.PanicLevel), zap.WithCaller(false))
+		log := app.Log.WithOptions(zap.WithCaller(false))
 
 		if app.Opt.Federation.Enabled {
 			log.Warn("Record Federation is still in EXPERIMENTAL phase")

app/servers.go

@@ -2,6 +2,12 @@ package app
 
 import (
 	"context"
+	"net/http"
+	"path"
+	"regexp"
+	"strings"
+	"sync"
+
 	automationRest "github.com/cortezaproject/corteza-server/automation/rest"
 	composeRest "github.com/cortezaproject/corteza-server/compose/rest"
 	"github.com/cortezaproject/corteza-server/docs"
@@ -15,11 +21,6 @@ import (
 	"github.com/cortezaproject/corteza-server/system/scim"
 	"github.com/go-chi/chi"
 	"go.uber.org/zap"
-	"net/http"
-	"path"
-	"regexp"
-	"strings"
-	"sync"
 )
 
 func (app *CortezaApp) Serve(ctx context.Context) (err error) {
@@ -58,7 +59,6 @@ func (app *CortezaApp) mountHttpRoutes(r chi.Router) {
 	func() {
 		if ho.WebappEnabled && ho.ApiEnabled && ho.ApiBaseUrl == ho.WebappBaseUrl {
 			app.Log.
-				WithOptions(zap.AddStacktrace(zap.PanicLevel)).
 				Warn("client web applications and api can not use the same base URL: '" + ho.WebappBaseUrl + "'")
 			ho.WebappEnabled = false
 		}
@@ -122,7 +122,6 @@ func (app *CortezaApp) mountHttpRoutes(r chi.Router) {
 
 		if app.Opt.SCIM.Secret == "" {
 			app.Log.
-				WithOptions(zap.AddStacktrace(zap.PanicLevel)).
 				Error("SCIM secret empty")
 		}
 

auth/auth.go

@@ -51,7 +51,7 @@ func New(ctx context.Context, log *zap.Logger, s store.Storer, opt options.AuthO
 		defClient *types.AuthClient
 	)
 
-	log = log.Named("auth").WithOptions(zap.AddStacktrace(zap.PanicLevel))
+	log = log.Named("auth")
 	ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_Auth)
 
 	svc = &service{

auth/oauth2/oauth2.go

@@ -1,6 +1,8 @@
 package oauth2
 
 import (
+	"strings"
+
 	"github.com/cortezaproject/corteza-server/pkg/logger"
 	"github.com/cortezaproject/corteza-server/pkg/options"
 	"github.com/dgrijalva/jwt-go"
@@ -9,7 +11,6 @@ import (
 	"github.com/go-oauth2/oauth2/v4/manage"
 	"github.com/go-oauth2/oauth2/v4/server"
 	"go.uber.org/zap"
-	"strings"
 )
 
 const (
@@ -90,7 +91,6 @@ func NewServer(manager *manage.Manager) *server.Server {
 		}
 
 		logger.Default().
-			WithOptions(zap.AddStacktrace(zap.PanicLevel)).
 			Error(msg)
 	})
 

automation/service/trigger.go

@@ -486,7 +486,6 @@ func (svc *trigger) registerTriggers(wf *types.Workflow, runAs auth.Identifiable
 		g         *wfexec.Graph
 		issues    types.WorkflowIssueSet
 		wfLog     = svc.log.
-				WithOptions(zap.AddStacktrace(zap.DPanicLevel)).
 				With(zap.Uint64("workflowID", wf.ID))
 
 			// register only enabled, undeleted workflows

compose/service/values/formatter.go

@@ -1,12 +1,13 @@
 package values
 
 import (
+	"strings"
+	"time"
+
 	"github.com/cortezaproject/corteza-server/compose/types"
 	"github.com/cortezaproject/corteza-server/pkg/expr"
 	"github.com/cortezaproject/corteza-server/pkg/logger"
 	"go.uber.org/zap"
-	"strings"
-	"time"
 )
 
 type (
@@ -25,7 +26,6 @@ func (f formatter) Run(m *types.Module, vv types.RecordValueSet) types.RecordVal
 		exprParser = expr.Parser()
 
 		log = logger.Default().
-			WithOptions(zap.AddStacktrace(zap.PanicLevel)).
 			With(zap.Uint64("module", m.ID))
 	)
 

compose/service/values/sanitizer.go

@@ -2,13 +2,14 @@ package values
 
 import (
 	"fmt"
-	"github.com/cortezaproject/corteza-server/pkg/expr"
-	"github.com/cortezaproject/corteza-server/pkg/logger"
-	"go.uber.org/zap"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/cortezaproject/corteza-server/pkg/expr"
+	"github.com/cortezaproject/corteza-server/pkg/logger"
+	"go.uber.org/zap"
+
 	"github.com/cortezaproject/corteza-server/compose/types"
 )
 
@@ -60,7 +61,6 @@ func (s sanitizer) Run(m *types.Module, vv types.RecordValueSet) (out types.Reco
 		kind string
 
 		log = logger.Default().
-			WithOptions(zap.AddStacktrace(zap.PanicLevel)).
 			With(zap.Uint64("module", m.ID))
 	)
 

pkg/corredor/service.go

@@ -655,7 +655,7 @@ func (svc service) exec(ctx context.Context, script string, runAs string, args S
 			zap.String("runAs", runAs),
 			zap.String("eventType", args.EventType()),
 			zap.String("resourceType", args.ResourceType()),
-		).WithOptions(zap.AddStacktrace(zap.FatalLevel))
+		)
 	)
 
 	log.Debug("triggered")

pkg/logger/logger.go

@@ -36,9 +36,9 @@ func MakeDebugLogger() *zap.Logger {
 		panic(err)
 	}
 
-	logger.Debug("full debug mode enabled (LOG_DEBUG=true), enjoy all the colors and verbosity")
+	logger.Debug("full debug mode enabled (LOG_DEBUG=true)")
 
-	return logger
+	return applyStacktrace(logger, zap.DPanicLevel)
 }
 
 func Init() {
@@ -75,6 +75,22 @@ func Init() {
 	if err != nil {
 		panic(err)
 	}
+
+	// Add stacktrace ONLY for panics
+	defaultLogger = applyStacktrace(defaultLogger, zap.PanicLevel)
+}
+
+// applies configured stacktrace level
+//
+// By default it uses Panic or DPanic (depends if debug logger is used)
+// This can be manipulated by setting LOG_STACKTRACE_LEVEL to
+// value "debug", "info", "warn", "error", "dpanic", "panic", or "fatal"
+func applyStacktrace(in *zap.Logger, def zapcore.Level) *zap.Logger {
+	if stl, has := os.LookupEnv("LOG_STACKTRACE_LEVEL"); has {
+		_ = def.UnmarshalText([]byte(stl))
+	}
+
+	return in.WithOptions(zap.AddStacktrace(def))
 }
 
 func Default() *zap.Logger {

pkg/monitor/monitor.go

@@ -32,7 +32,7 @@ var (
 )
 
 func Setup(logger *zap.Logger, o options.MonitorOpt) {
-	log = logger.Named("monitor").WithOptions(zap.AddStacktrace(zap.PanicLevel))
+	log = logger.Named("monitor")
 	opt = o
 }
 

pkg/provision/provision.go

@@ -2,6 +2,8 @@ package provision
 
 import (
 	"context"
+	"time"
+
 	"github.com/cortezaproject/corteza-server/pkg/errors"
 	"github.com/cortezaproject/corteza-server/pkg/id"
 	"github.com/cortezaproject/corteza-server/pkg/options"
@@ -10,12 +12,9 @@ import (
 	"github.com/cortezaproject/corteza-server/system/service"
 	"github.com/cortezaproject/corteza-server/system/types"
 	"go.uber.org/zap"
-	"time"
 )
 
 func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt options.ProvisionOpt, authOpt options.AuthOpt) error {
-	log = log.WithOptions(zap.AddStacktrace(zap.PanicLevel))
-
 	ffn := []func() error{
 		func() error { return roles(ctx, s) },
 

pkg/webapp/serve.go

@@ -3,16 +3,17 @@ package webapp
 import (
 	"bytes"
 	"fmt"
-	"github.com/cortezaproject/corteza-server/pkg/logger"
-	"github.com/cortezaproject/corteza-server/pkg/options"
-	"github.com/go-chi/chi"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
 	"os"
 	"path"
 	"strings"
 	"time"
+
+	"github.com/cortezaproject/corteza-server/pkg/logger"
+	"github.com/cortezaproject/corteza-server/pkg/options"
+	"github.com/go-chi/chi"
+	"go.uber.org/zap"
 )
 
 var (
@@ -90,7 +91,7 @@ func serveIndex(opt options.HTTPServerOpt, indexHTML []byte, serve http.Handler)
 			return
 		}
 
-		logger.Default().WithOptions(zap.AddStacktrace(zap.PanicLevel)).Error(
+		logger.Default().Error(
 			"failed to serve static file",
 			zap.Error(err),
 			zap.Stringer("url", r.URL),

pkg/websocket/server.go

@@ -1,6 +1,10 @@
 package websocket
 
 import (
+	"io"
+	"net/http"
+	"sync"
+
 	"github.com/cortezaproject/corteza-server/pkg/auth"
 	"github.com/cortezaproject/corteza-server/pkg/errors"
 	"github.com/cortezaproject/corteza-server/pkg/options"
@@ -8,9 +12,6 @@ import (
 	"github.com/dgrijalva/jwt-go"
 	"github.com/gorilla/websocket"
 	"go.uber.org/zap"
-	"io"
-	"net/http"
-	"sync"
 )
 
 var (
@@ -48,7 +49,7 @@ func Server(logger *zap.Logger, config options.WebsocketOpt) *server {
 
 	return &server{
 		config:      config,
-		logger:      logger.WithOptions(zap.AddStacktrace(zap.PanicLevel)).Named("websocket"),
+		logger:      logger.Named("websocket"),
 		accessToken: auth.DefaultJwtHandler,
 		sessions:    make(map[uint64]map[uint64]io.Writer),
 	}

pkg/wfexec/session.go

@@ -3,14 +3,15 @@ package wfexec
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
+	"sync"
+	"time"
+
 	"github.com/cortezaproject/corteza-server/pkg/auth"
 	"github.com/cortezaproject/corteza-server/pkg/expr"
 	"github.com/cortezaproject/corteza-server/pkg/id"
 	"github.com/cortezaproject/corteza-server/pkg/logger"
 	"go.uber.org/zap"
-	"runtime/debug"
-	"sync"
-	"time"
 )
 
 type (
@@ -181,7 +182,6 @@ func NewSession(ctx context.Context, g *Graph, oo ...SessionOpt) *Session {
 	}
 
 	s.log = s.log.
-		WithOptions(zap.AddStacktrace(zap.ErrorLevel)).
 		With(zap.Uint64("sessionID", s.id))
 
 	go s.worker(ctx)
@@ -566,7 +566,7 @@ func (s *Session) exec(ctx context.Context, st *State) (err error) {
 		} else {
 			// push logger to context but raise the stacktrace level to panic
 			// to prevent overly verbose traces
-			ctx = logger.ContextWithValue(ctx, log.WithOptions(zap.AddStacktrace(zap.PanicLevel)))
+			ctx = logger.ContextWithValue(ctx, log)
 
 			// Context received in exec() wil not have the identity we're expecting
 			// so we need to pull it from state owner and add it to new context

store/rdbms/instrumentation/debug.go

@@ -5,10 +5,11 @@ import (
 	"database/sql/driver"
 	"errors"
 	"fmt"
+	"time"
+
 	"github.com/cortezaproject/corteza-server/pkg/logger"
 	"github.com/ngrok/sqlmw"
 	"go.uber.org/zap"
-	"time"
 )
 
 type (
@@ -44,8 +45,6 @@ func (ld debug) log(ctx context.Context) *zap.Logger {
 			//       have the same stack trace, might
 			//       need adjustment on individual fn bellow
 			zap.AddCallerSkip(9),
-			// never add stacktrace to these logs
-			zap.AddStacktrace(zap.FatalLevel),
 		)
 }
 

store/rdbms/log.go

@@ -2,6 +2,7 @@ package rdbms
 
 import (
 	"context"
+
 	"github.com/cortezaproject/corteza-server/pkg/logger"
 	"go.uber.org/zap"
 )
@@ -12,9 +13,7 @@ import (
 func (s Store) log(ctx context.Context) *zap.Logger {
 	return logger.ContextValue(ctx, s.logger).
 		Named("store.rdbms").
-		WithOptions(zap.AddCallerSkip(2)).
-		// Really not interested in call stack
-		WithOptions(zap.AddStacktrace(zap.FatalLevel))
+		WithOptions(zap.AddCallerSkip(2))
 }
 
 func (s Store) SetLogger(logger *zap.Logger) {