DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Optimized boot levels order & logic

Browse Source
This commit is contained in:
Denis Arh
2021-07-08 11:23:18 +02:00
parent 46f86dbd21
commit 48e1d0e4aa
8 changed files with 134 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/app.go
+5 -2
View File
@@ -61,8 +61,11 @@ type (
)
func New() *CortezaApp {
app := &CortezaApp{lvl: bootLevelWaiting}
app.Opt = options.Init()
app := &CortezaApp{
lvl: bootLevelWaiting,
Log: zap.NewNop(),
}
app.InitCLI()
return app
}
app/boot_levels.go
+40 -43
View File
@@ -6,10 +6,9 @@ import (
"fmt"
"strings"
authService "github.com/cortezaproject/corteza-server/auth"
authHandlers "github.com/cortezaproject/corteza-server/auth/handlers"
"github.com/cortezaproject/corteza-server/auth/saml"
authService "github.com/cortezaproject/corteza-server/auth"
authSettings "github.com/cortezaproject/corteza-server/auth/settings"
autService "github.com/cortezaproject/corteza-server/automation/service"
cmpService "github.com/cortezaproject/corteza-server/compose/service"
@@ -44,9 +43,8 @@ const (
bootLevelWaiting = iota
bootLevelSetup
bootLevelStoreInitialized
bootLevelServicesInitialized
bootLevelUpgraded
bootLevelProvisioned
bootLevelServicesInitialized
bootLevelActivated
)
@@ -246,11 +244,41 @@ func (app *CortezaApp) InitStore(ctx context.Context) (err error) {
return nil
}
// Provision instance with configuration and settings
// by importing preset configurations and running autodiscovery procedures
func (app *CortezaApp) Provision(ctx context.Context) (err error) {
if app.lvl >= bootLevelProvisioned {
return
}
if err = app.InitStore(ctx); err != nil {
return err
}
if !app.Opt.Provision.Always {
app.Log.Debug("provisioning skipped (PROVISION_ALWAYS=false)")
} else {
defer sentry.Recover()
ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_APP_Provision)
ctx = auth.SetSuperUserContext(ctx)
if err = provision.Run(ctx, app.Log, app.Store, app.Opt.Provision, app.Opt.Auth); err != nil {
return err
}
}
app.lvl = bootLevelProvisioned
return
}
// InitServices initializes all services used
func (app *CortezaApp) InitServices(ctx context.Context) (err error) {
if app.lvl >= bootLevelServicesInitialized {
return nil
} else if err := app.InitStore(ctx); err != nil {
}
if err := app.Provision(ctx); err != nil {
return err
}
@@ -264,14 +292,12 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) {
}
{
// Initialize RBAC subsystem
//Initialize RBAC subsystem
// and (re)load rules from the storage backend
err = rbac.Initialize(app.Log, app.Store)
if err != nil {
return
}
ac := rbac.NewService(app.Log, app.Store)
ac.Reload(ctx)
rbac.Global().Reload(ctx)
rbac.SetGlobal(ac)
}
if app.Opt.Messagebus.Enabled {
@@ -344,42 +370,13 @@ func (app *CortezaApp) InitServices(ctx context.Context) (err error) {
return
}
// Provision instance with configuration and settings
// by importing preset configurations and running autodiscovery procedures
func (app *CortezaApp) Provision(ctx context.Context) (err error) {
if app.lvl >= bootLevelProvisioned {
return
}
if err = app.InitServices(ctx); err != nil {
return err
}
if !app.Opt.Provision.Always {
app.Log.Debug("provisioning skipped (PROVISION_ALWAYS=false)")
} else {
defer sentry.Recover()
ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_APP_Provision)
ctx = auth.SetSuperUserContext(ctx)
if err = provision.Run(ctx, app.Log, app.Store, app.Opt.Provision, app.Opt.Auth); err != nil {
return err
}
// Provisioning doesn't automatically reload rbac rules, so this is required
rbac.Global().Reload(ctx)
}
app.lvl = bootLevelProvisioned
return
}
// Activate start all internal services and watchers
func (app *CortezaApp) Activate(ctx context.Context) (err error) {
if app.lvl >= bootLevelActivated {
return
} else if err := app.Provision(ctx); err != nil {
}
if err := app.InitServices(ctx); err != nil {
return err
}
app/cli.go
+1 -1
View File
@@ -91,7 +91,7 @@ func (app *CortezaApp) InitCLI() {
serveCmd,
upgradeCmd,
provisionCmd,
authCommands.Command(app),
authCommands.Command(app, storeInit),
federationCommands.Sync(app),
cli.EnvCommand(),
cli.VersionCommand(),
auth/commands/commands.go
+9 -4
View File
@@ -7,6 +7,7 @@ import (
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/cli"
"github.com/cortezaproject/corteza-server/pkg/options"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/spf13/cobra"
@@ -25,8 +26,7 @@ func commandPreRunInitService(app serviceInitializer) func(*cobra.Command, []str
}
}
// Will perform OpenID connect auto-configuration
func Command(app serviceInitializer) *cobra.Command {
func Command(app serviceInitializer, storeInit func(ctx context.Context) (store.Storer, error)) *cobra.Command {
var (
enableDiscoveredProvider bool
skipValidationOnAutoDiscoveredProvider bool
@@ -43,9 +43,14 @@ func Command(app serviceInitializer) *cobra.Command {
Args: cobra.ExactArgs(2),
PreRunE: commandPreRunInitService(app),
Run: func(cmd *cobra.Command, args []string) {
ctx := auth.SetSuperUserContext(cli.Context())
_, err := external.RegisterOidcProvider(
ctx := cli.Context()
s, err := storeInit(ctx)
cli.HandleError(err)
_, err = external.RegisterOidcProvider(
ctx,
s,
app.Options().Auth,
args[0],
args[1],
auth/external/register.go
Vendored
+41 -28
View File
@@ -2,26 +2,28 @@ package external
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/options"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/crusttech/go-oidc"
"github.com/pkg/errors"
"go.uber.org/zap"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/cortezaproject/corteza-server/pkg/options"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/crusttech/go-oidc"
"github.com/pkg/errors"
"go.uber.org/zap"
)
func AddProvider(ctx context.Context, eap *types.ExternalAuthProvider, force bool) error {
var (
s = service.CurrentSettings
log = log.With(
zap.Bool("force", force),
zap.String("handle", eap.Handle),
zap.String("key", eap.Key),
)
// AddProvider is used by provisioning process
func AddProvider(ctx context.Context, s store.Settings, eap *types.ExternalAuthProvider, force bool) error {
prefix := "auth.external.providers." + eap.Key + "."
log := log.With(
zap.Bool("force", force),
zap.String("handle", eap.Handle),
zap.String("key", eap.Key),
)
if eap.IssuerUrl != "" {
@@ -30,18 +32,27 @@ func AddProvider(ctx context.Context, eap *types.ExternalAuthProvider, force boo
log.Info("adding external authentication provider")
ss, _, err := store.SearchSettings(ctx, s, types.SettingsFilter{
Prefix: prefix,
})
if err != nil {
return err
}
ex := ss.KV().CutPrefix(prefix)
if !force {
if ex := s.Auth.External.Providers.FindByHandle(eap.Handle); ex != nil && ex.Key == eap.Key && ex.Secret == eap.Secret {
// check if exists before storing it
if len(ex) > 0 && ex.String("key") == eap.Key && ex.String("secret") == eap.Secret {
return nil
}
}
if vv, err := eap.EncodeKV(); err != nil {
log.Error("could not prepare settings", zap.Error(err))
return err
} else if err = service.DefaultSettings.BulkSet(ctx, vv); err != nil {
log.Error("could not store settings", zap.Error(err))
return fmt.Errorf("could not encode auth provider values: %w", err)
return err
} else if err = store.UpsertSetting(ctx, s, vv...); err != nil {
return fmt.Errorf("could not store auth provider values: %w", err)
}
log.Info("external authentication provider added")
@@ -90,14 +101,16 @@ func DiscoverOidcProvider(ctx context.Context, opt options.AuthOpt, name, url st
return
}
func RegisterOidcProvider(ctx context.Context, opt options.AuthOpt, name, providerUrl string, force, validate, enable bool) (eap *types.ExternalAuthProvider, err error) {
var (
s = service.CurrentSettings
)
func RegisterOidcProvider(ctx context.Context, s store.Settings, opt options.AuthOpt, name, providerUrl string, force, validate, enable bool) (eap *types.ExternalAuthProvider, err error) {
if !force {
if s.Auth.External.Providers.FindByHandle(OIDC_PROVIDER_PREFIX+name) != nil {
return
prefix := "auth.external.providers." + eap.Key + "."
vv, _, err := store.SearchSettings(ctx, s, types.SettingsFilter{
Prefix: prefix,
})
if err != nil || len(vv) > 0 {
return nil, err
}
}
@@ -154,7 +167,7 @@ func RegisterOidcProvider(ctx context.Context, opt options.AuthOpt, name, provid
vv = append(vv, v)
}
err = service.DefaultSettings.BulkSet(ctx, vv)
err = store.UpsertSetting(ctx, s, vv...)
if err != nil {
return
}
pkg/provision/auth.go
+27 -44
View File
@@ -2,52 +2,29 @@ package provision
import (
"context"
"fmt"
"os"
"github.com/cortezaproject/corteza-server/pkg/errors"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/spf13/cast"
"go.uber.org/zap"
"os"
)
type (
settingsService interface {
FindByPrefix(context.Context, ...string) (types.SettingValueSet, error)
BulkSet(context.Context, types.SettingValueSet) error
}
)
var (
IsMonolith bool
)
// Discovers "auth.%" settings from the environment
//
// when other kinds of auto-discoverable settings come, lambdas inside will probably need a bit of refactoring
func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc settingsService) (err error) {
func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
type (
stringWrapper func() string
boolWrapper func() bool
)
var (
current types.SettingValueSet
)
if log == nil {
log = zap.NewNop()
}
log = service.DefaultLogger.Named("auth-settings-discovery")
current, err = svc.FindByPrefix(ctx, "auth.")
if err != nil {
return
}
log = log.Named("auth.settings-discovery")
var (
new = current
// Setter
//
// Finds existing settings, tries with environmental "PROVISION_SETTINGS_AUTH_..." probing
@@ -57,25 +34,32 @@ func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc setting
// how settings were discovered and set
//
// @todo generalize and move under settings
set = func(name string, env string, def interface{}, maskSensitive bool) {
set = func(name string, env string, def interface{}, maskSensitive bool) error {
var (
log = log.With(
zap.String("name", name),
)
v = current.First(name)
value interface{}
envExists bool
value interface{}
v, err = s.LookupSettingByNameOwnedBy(ctx, name, 0)
)
if !errors.IsNotFound(err) && err != nil {
return fmt.Errorf("could not load settings value for '%s': %w", name, err)
}
if v != nil {
// Nothing to discover, already set
log.Debug("already set", logger.MaskIf("value", v, maskSensitive))
return
return nil
}
v = &types.SettingValue{Name: name}
value, envExists := os.LookupEnv(env)
value, envExists = os.LookupEnv(env)
switch dfn := def.(type) {
case stringWrapper:
@@ -99,18 +83,15 @@ func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc setting
}
default:
log.Error("unsupported type")
return
return fmt.Errorf("unsupported type %T for '%s'", def, name)
}
if err := v.SetValue(value); err != nil {
log.Error("could not set value", zap.Error(err))
return
return fmt.Errorf("could not set value to '%q': %w", name, err)
}
log.Info("value auto-discovered")
new.Replace(v)
log.Debug("value auto-discovered")
return s.UpsertSetting(ctx, v)
}
// Assume we have emailing capabilities if SMTP_HOST variable is set
@@ -204,8 +185,10 @@ func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc setting
}
for _, item := range list {
set(item.nme, item.env, item.def, item.mask)
if err = set(item.nme, item.env, item.def, item.mask); err != nil {
return err
}
}
return svc.BulkSet(ctx, new)
return nil
}
pkg/provision/oidc.go
+8 -6
View File
@@ -3,19 +3,21 @@ package provision
import (
"context"
"fmt"
"os"
"strings"
"github.com/cortezaproject/corteza-server/auth/external"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/options"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
"os"
"strings"
)
// Provisions OIDC providers from PROVISION_OIDC_PROVIDER env variable
//
// Env variable should contains space delimited pairs of providers (<name> <provider> ....)
func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, opt options.AuthOpt) (err error) {
func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, s store.Settings, opt options.AuthOpt) (err error) {
var provider = strings.TrimSpace(options.EnvString("PROVISION_OIDC_PROVIDER", ""))
log.Debug("OIDC auto discovery provision",
@@ -49,7 +51,7 @@ func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, opt options.AuthOpt
//
// enable: true
// we want provider & the entire external auth to be validated
eap, err = external.RegisterOidcProvider(ctx, opt, name, purl, false, false, true)
eap, err = external.RegisterOidcProvider(ctx, s, opt, name, purl, false, false, true)
if err != nil {
log.Error(
@@ -72,7 +74,7 @@ func oidcAutoDiscovery(ctx context.Context, log *zap.Logger, opt options.AuthOpt
return
}
func authAddExternals(ctx context.Context, log *zap.Logger) (err error) {
func authAddExternals(ctx context.Context, log *zap.Logger, s store.Settings) (err error) {
var (
kinds = []string{
"github",
@@ -127,7 +129,7 @@ func authAddExternals(ctx context.Context, log *zap.Logger) (err error) {
ctx = auth.SetSuperUserContext(ctx)
_ = external.AddProvider(ctx, eap, false)
_ = external.AddProvider(ctx, s, eap, false)
}
return
pkg/provision/provision.go
+3 -5
View File
@@ -9,7 +9,6 @@ import (
"github.com/cortezaproject/corteza-server/pkg/options"
"github.com/cortezaproject/corteza-server/pkg/rand"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
)
@@ -26,10 +25,9 @@ func Run(ctx context.Context, log *zap.Logger, s store.Storer, provisionOpt opti
func() error { return importConfig(ctx, log, s, provisionOpt.Path) },
// Auto-discoveries and other parts that cannot be imported from static files
func() error { return authSettingsAutoDiscovery(ctx, log, service.DefaultSettings) },
func() error { return authAddExternals(ctx, log) },
func() error { return service.DefaultSettings.UpdateCurrent(ctx) },
func() error { return oidcAutoDiscovery(ctx, log, authOpt) },
func() error { return authSettingsAutoDiscovery(ctx, log, s) },
func() error { return authAddExternals(ctx, log, s) },
func() error { return oidcAutoDiscovery(ctx, log, s, authOpt) },
func() error { return defaultAuthClient(ctx, log, s, authOpt) },
}