From 48e1d0e4aa54353560bcbc0dee0df9018bc942fb Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Sun, 23 May 2021 09:25:50 +0200 Subject: [PATCH] Optimized boot levels order & logic --- app/app.go | 7 +++- app/boot_levels.go | 83 ++++++++++++++++++-------------------- app/cli.go | 2 +- auth/commands/commands.go | 13 ++++-- auth/external/register.go | 69 ++++++++++++++++++------------- pkg/provision/auth.go | 71 +++++++++++++------------------- pkg/provision/oidc.go | 14 ++++--- pkg/provision/provision.go | 8 ++-- 8 files changed, 134 insertions(+), 133 deletions(-) diff --git a/app/app.go b/app/app.go index 82e763469..0537762d2 100644 --- a/app/app.go +++ b/app/app.go @@ -61,8 +61,11 @@ type ( ) func New() *CortezaApp { - app := &CortezaApp{lvl: bootLevelWaiting} - app.Opt = options.Init() + app := &CortezaApp{ + lvl: bootLevelWaiting, + Log: zap.NewNop(), + } + app.InitCLI() return app } diff --git a/app/boot_levels.go b/app/boot_levels.go index c69da2b6f..b15ce087b 100644 --- a/app/boot_levels.go +++ b/app/boot_levels.go @@ -6,10 +6,9 @@ import ( "fmt" "strings" + authService "github.com/cortezaproject/corteza-server/auth" authHandlers "github.com/cortezaproject/corteza-server/auth/handlers" "github.com/cortezaproject/corteza-server/auth/saml" - - authService "github.com/cortezaproject/corteza-server/auth" authSettings "github.com/cortezaproject/corteza-server/auth/settings" autService "github.com/cortezaproject/corteza-server/automation/service" cmpService "github.com/cortezaproject/corteza-server/compose/service" @@ -44,9 +43,8 @@ const ( bootLevelWaiting = iota bootLevelSetup bootLevelStoreInitialized - bootLevelServicesInitialized - bootLevelUpgraded bootLevelProvisioned + bootLevelServicesInitialized bootLevelActivated ) @@ -246,11 +244,41 @@ func (app *CortezaApp) InitStore(ctx context.Context) (err error) { return nil } +// Provision instance with configuration and settings +// by importing preset configurations and running autodiscovery procedures +func (app *CortezaApp) Provision(ctx context.Context) (err error) { + if app.lvl >= bootLevelProvisioned { + return + } + + if err = app.InitStore(ctx); err != nil { + return err + } + + if !app.Opt.Provision.Always { + app.Log.Debug("provisioning skipped (PROVISION_ALWAYS=false)") + } else { + defer sentry.Recover() + + ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_APP_Provision) + ctx = auth.SetSuperUserContext(ctx) + + if err = provision.Run(ctx, app.Log, app.Store, app.Opt.Provision, app.Opt.Auth); err != nil { + return err + } + } + + app.lvl = bootLevelProvisioned + return +} + // InitServices initializes all services used func (app *CortezaApp) InitServices(ctx context.Context) (err error) { if app.lvl >= bootLevelServicesInitialized { return nil - } else if err := app.InitStore(ctx); err != nil { + } + + if err := app.Provision(ctx); err != nil { return err } @@ -264,14 +292,12 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) { } { - // Initialize RBAC subsystem + //Initialize RBAC subsystem // and (re)load rules from the storage backend - err = rbac.Initialize(app.Log, app.Store) - if err != nil { - return - } + ac := rbac.NewService(app.Log, app.Store) + ac.Reload(ctx) - rbac.Global().Reload(ctx) + rbac.SetGlobal(ac) } if app.Opt.Messagebus.Enabled { @@ -344,42 +370,13 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) { return } -// Provision instance with configuration and settings -// by importing preset configurations and running autodiscovery procedures -func (app *CortezaApp) Provision(ctx context.Context) (err error) { - if app.lvl >= bootLevelProvisioned { - return - } - - if err = app.InitServices(ctx); err != nil { - return err - } - - if !app.Opt.Provision.Always { - app.Log.Debug("provisioning skipped (PROVISION_ALWAYS=false)") - } else { - defer sentry.Recover() - - ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_APP_Provision) - ctx = auth.SetSuperUserContext(ctx) - - if err = provision.Run(ctx, app.Log, app.Store, app.Opt.Provision, app.Opt.Auth); err != nil { - return err - } - - // Provisioning doesn't automatically reload rbac rules, so this is required - rbac.Global().Reload(ctx) - } - - app.lvl = bootLevelProvisioned - return -} - // Activate start all internal services and watchers func (app *CortezaApp) Activate(ctx context.Context) (err error) { if app.lvl >= bootLevelActivated { return - } else if err := app.Provision(ctx); err != nil { + } + + if err := app.InitServices(ctx); err != nil { return err } diff --git a/app/cli.go b/app/cli.go index 0257ca6eb..5697bd692 100644 --- a/app/cli.go +++ b/app/cli.go @@ -91,7 +91,7 @@ func (app *CortezaApp) InitCLI() { serveCmd, upgradeCmd, provisionCmd, - authCommands.Command(app), + authCommands.Command(app, storeInit), federationCommands.Sync(app), cli.EnvCommand(), cli.VersionCommand(), diff --git a/auth/commands/commands.go b/auth/commands/commands.go index 219257d01..39576f854 100644 --- a/auth/commands/commands.go +++ b/auth/commands/commands.go @@ -7,6 +7,7 @@ import ( "github.com/cortezaproject/corteza-server/pkg/auth" "github.com/cortezaproject/corteza-server/pkg/cli" "github.com/cortezaproject/corteza-server/pkg/options" + "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/service" "github.com/cortezaproject/corteza-server/system/types" "github.com/spf13/cobra" @@ -25,8 +26,7 @@ func commandPreRunInitService(app serviceInitializer) func(*cobra.Command, []str } } -// Will perform OpenID connect auto-configuration -func Command(app serviceInitializer) *cobra.Command { +func Command(app serviceInitializer, storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Command { var ( enableDiscoveredProvider bool skipValidationOnAutoDiscoveredProvider bool @@ -43,9 +43,14 @@ func Command(app serviceInitializer) *cobra.Command { Args: cobra.ExactArgs(2), PreRunE: commandPreRunInitService(app), Run: func(cmd *cobra.Command, args []string) { - ctx := auth.SetSuperUserContext(cli.Context()) - _, err := external.RegisterOidcProvider( + ctx := cli.Context() + + s, err := storeInit(ctx) + cli.HandleError(err) + + _, err = external.RegisterOidcProvider( ctx, + s, app.Options().Auth, args[0], args[1], diff --git a/auth/external/register.go b/auth/external/register.go index ae0200290..496fe1afc 100644 --- a/auth/external/register.go +++ b/auth/external/register.go @@ -2,26 +2,28 @@ package external import ( "context" - "github.com/cortezaproject/corteza-server/pkg/options" - "github.com/cortezaproject/corteza-server/system/service" - "github.com/cortezaproject/corteza-server/system/types" - "github.com/crusttech/go-oidc" - "github.com/pkg/errors" - "go.uber.org/zap" + "fmt" "io/ioutil" "net/http" "net/url" "strings" + + "github.com/cortezaproject/corteza-server/pkg/options" + "github.com/cortezaproject/corteza-server/store" + "github.com/cortezaproject/corteza-server/system/types" + "github.com/crusttech/go-oidc" + "github.com/pkg/errors" + "go.uber.org/zap" ) -func AddProvider(ctx context.Context, eap *types.ExternalAuthProvider, force bool) error { - var ( - s = service.CurrentSettings - log = log.With( - zap.Bool("force", force), - zap.String("handle", eap.Handle), - zap.String("key", eap.Key), - ) +// AddProvider is used by provisioning process +func AddProvider(ctx context.Context, s store.Settings, eap *types.ExternalAuthProvider, force bool) error { + prefix := "auth.external.providers." + eap.Key + "." + + log := log.With( + zap.Bool("force", force), + zap.String("handle", eap.Handle), + zap.String("key", eap.Key), ) if eap.IssuerUrl != "" { @@ -30,18 +32,27 @@ func AddProvider(ctx context.Context, eap *types.ExternalAuthProvider, force boo log.Info("adding external authentication provider") + ss, _, err := store.SearchSettings(ctx, s, types.SettingsFilter{ + Prefix: prefix, + }) + + if err != nil { + return err + } + + ex := ss.KV().CutPrefix(prefix) + if !force { - if ex := s.Auth.External.Providers.FindByHandle(eap.Handle); ex != nil && ex.Key == eap.Key && ex.Secret == eap.Secret { + // check if exists before storing it + if len(ex) > 0 && ex.String("key") == eap.Key && ex.String("secret") == eap.Secret { return nil } } - if vv, err := eap.EncodeKV(); err != nil { - log.Error("could not prepare settings", zap.Error(err)) - return err - } else if err = service.DefaultSettings.BulkSet(ctx, vv); err != nil { - log.Error("could not store settings", zap.Error(err)) + return fmt.Errorf("could not encode auth provider values: %w", err) return err + } else if err = store.UpsertSetting(ctx, s, vv...); err != nil { + return fmt.Errorf("could not store auth provider values: %w", err) } log.Info("external authentication provider added") @@ -90,14 +101,16 @@ func DiscoverOidcProvider(ctx context.Context, opt options.AuthOpt, name, url st return } -func RegisterOidcProvider(ctx context.Context, opt options.AuthOpt, name, providerUrl string, force, validate, enable bool) (eap *types.ExternalAuthProvider, err error) { - var ( - s = service.CurrentSettings - ) - +func RegisterOidcProvider(ctx context.Context, s store.Settings, opt options.AuthOpt, name, providerUrl string, force, validate, enable bool) (eap *types.ExternalAuthProvider, err error) { if !force { - if s.Auth.External.Providers.FindByHandle(OIDC_PROVIDER_PREFIX+name) != nil { - return + prefix := "auth.external.providers." + eap.Key + "." + + vv, _, err := store.SearchSettings(ctx, s, types.SettingsFilter{ + Prefix: prefix, + }) + + if err != nil || len(vv) > 0 { + return nil, err } } @@ -154,7 +167,7 @@ func RegisterOidcProvider(ctx context.Context, opt options.AuthOpt, name, provid vv = append(vv, v) } - err = service.DefaultSettings.BulkSet(ctx, vv) + err = store.UpsertSetting(ctx, s, vv...) if err != nil { return } diff --git a/pkg/provision/auth.go b/pkg/provision/auth.go index fd2a8960d..3496fefdc 100644 --- a/pkg/provision/auth.go +++ b/pkg/provision/auth.go @@ -2,52 +2,29 @@ package provision import ( "context" + "fmt" + "os" + + "github.com/cortezaproject/corteza-server/pkg/errors" "github.com/cortezaproject/corteza-server/pkg/logger" - "github.com/cortezaproject/corteza-server/system/service" + "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/types" "github.com/spf13/cast" "go.uber.org/zap" - "os" -) - -type ( - settingsService interface { - FindByPrefix(context.Context, ...string) (types.SettingValueSet, error) - BulkSet(context.Context, types.SettingValueSet) error - } -) - -var ( - IsMonolith bool ) // Discovers "auth.%" settings from the environment // // when other kinds of auto-discoverable settings come, lambdas inside will probably need a bit of refactoring -func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc settingsService) (err error) { +func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, s store.Storer) (err error) { type ( stringWrapper func() string boolWrapper func() bool ) - var ( - current types.SettingValueSet - ) - - if log == nil { - log = zap.NewNop() - } - - log = service.DefaultLogger.Named("auth-settings-discovery") - - current, err = svc.FindByPrefix(ctx, "auth.") - if err != nil { - return - } + log = log.Named("auth.settings-discovery") var ( - new = current - // Setter // // Finds existing settings, tries with environmental "PROVISION_SETTINGS_AUTH_..." probing @@ -57,25 +34,32 @@ func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc setting // how settings were discovered and set // // @todo generalize and move under settings - set = func(name string, env string, def interface{}, maskSensitive bool) { + set = func(name string, env string, def interface{}, maskSensitive bool) error { + var ( log = log.With( zap.String("name", name), ) - v = current.First(name) - value interface{} + envExists bool + value interface{} + + v, err = s.LookupSettingByNameOwnedBy(ctx, name, 0) ) + if !errors.IsNotFound(err) && err != nil { + return fmt.Errorf("could not load settings value for '%s': %w", name, err) + } + if v != nil { // Nothing to discover, already set log.Debug("already set", logger.MaskIf("value", v, maskSensitive)) - return + return nil } v = &types.SettingValue{Name: name} - value, envExists := os.LookupEnv(env) + value, envExists = os.LookupEnv(env) switch dfn := def.(type) { case stringWrapper: @@ -99,18 +83,15 @@ func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc setting } default: - log.Error("unsupported type") - return + return fmt.Errorf("unsupported type %T for '%s'", def, name) } if err := v.SetValue(value); err != nil { - log.Error("could not set value", zap.Error(err)) - return + return fmt.Errorf("could not set value to '%q': %w", name, err) } - log.Info("value auto-discovered") - - new.Replace(v) + log.Debug("value auto-discovered") + return s.UpsertSetting(ctx, v) } // Assume we have emailing capabilities if SMTP_HOST variable is set @@ -204,8 +185,10 @@ func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc setting } for _, item := range list { - set(item.nme, item.env, item.def, item.mask) + if err = set(item.nme, item.env, item.def, item.mask); err != nil { + return err + } } - return svc.BulkSet(ctx, new) + return nil } diff --git a/pkg/provision/oidc.go b/pkg/provision/oidc.go index 7de162c52..9d4bc531b 100644 --- a/pkg/provision/oidc.go +++ b/pkg/provision/oidc.go @@ -3,19 +3,21 @@ package provision import ( "context" "fmt" + "os" + "strings" + "github.com/cortezaproject/corteza-server/auth/external" "github.com/cortezaproject/corteza-server/pkg/auth" "github.com/cortezaproject/corteza-server/pkg/options" + "github.com/cortezaproject/corteza-server/store" "github.com/cortezaproject/corteza-server/system/types" "go.uber.org/zap" - "os" - "strings" ) // Provisions OIDC providers from PROVISION_OIDC_PROVIDER env variable // // Env variable should contains space delimited pairs of providers ( ....) -func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, opt options.AuthOpt) (err error) { +func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, s store.Settings, opt options.AuthOpt) (err error) { var provider = strings.TrimSpace(options.EnvString("PROVISION_OIDC_PROVIDER", "")) log.Debug("OIDC auto discovery provision", @@ -49,7 +51,7 @@ func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, opt options.AuthOpt // // enable: true // we want provider & the entire external auth to be validated - eap, err = external.RegisterOidcProvider(ctx, opt, name, purl, false, false, true) + eap, err = external.RegisterOidcProvider(ctx, s, opt, name, purl, false, false, true) if err != nil { log.Error( @@ -72,7 +74,7 @@ func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, opt options.AuthOpt return } -func authAddExternals(ctx context.Context, log *zap.Logger) (err error) { +func authAddExternals(ctx context.Context, log *zap.Logger, s store.Settings) (err error) { var ( kinds = []string{ "github", @@ -127,7 +129,7 @@ func authAddExternals(ctx context.Context, log *zap.Logger) (err error) { ctx = auth.SetSuperUserContext(ctx) - _ = external.AddProvider(ctx, eap, false) + _ = external.AddProvider(ctx, s, eap, false) } return diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go index e8ae5f882..b556c5a14 100644 --- a/pkg/provision/provision.go +++ b/pkg/provision/provision.go @@ -9,7 +9,6 @@ import ( "github.com/cortezaproject/corteza-server/pkg/options" "github.com/cortezaproject/corteza-server/pkg/rand" "github.com/cortezaproject/corteza-server/store" - "github.com/cortezaproject/corteza-server/system/service" "github.com/cortezaproject/corteza-server/system/types" "go.uber.org/zap" ) @@ -26,10 +25,9 @@ func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt opti func() error { return importConfig(ctx, log, s, provisionOpt.Path) }, // Auto-discoveries and other parts that cannot be imported from static files - func() error { return authSettingsAutoDiscovery(ctx, log, service.DefaultSettings) }, - func() error { return authAddExternals(ctx, log) }, - func() error { return service.DefaultSettings.UpdateCurrent(ctx) }, - func() error { return oidcAutoDiscovery(ctx, log, authOpt) }, + func() error { return authSettingsAutoDiscovery(ctx, log, s) }, + func() error { return authAddExternals(ctx, log, s) }, + func() error { return oidcAutoDiscovery(ctx, log, s, authOpt) }, func() error { return defaultAuthClient(ctx, log, s, authOpt) }, }