From 63bfb182593afad73e0ba131921542a8980b91d3 Mon Sep 17 00:00:00 2001 From: Denis Arh Date: Tue, 29 Oct 2019 12:51:00 +0100 Subject: [PATCH] Add ability to provision (partial) unexisting settings We moved a couple of strings from code to provision files and we need to handle existing systems accordingly --- pkg/settings/importer.go | 4 +++ pkg/settings/types.go | 15 ++++++++- system/importer/importer.go | 54 +++++++++++++++++++++--------- system/provision-config.go | 66 +++++++++++++++++++++++++++++++++++-- system/service/settings.go | 6 +++- 5 files changed, 124 insertions(+), 21 deletions(-) diff --git a/pkg/settings/importer.go b/pkg/settings/importer.go index 33c319b17..44079bbe7 100644 --- a/pkg/settings/importer.go +++ b/pkg/settings/importer.go @@ -54,3 +54,7 @@ func (imp *Importer) addSetting(name string, value interface{}) (err error) { func (imp *Importer) Store(ctx context.Context, k ImportKeeper) (err error) { return k.BulkSet(imp.settings) } + +func (imp *Importer) GetValues() ValueSet { + return imp.settings +} diff --git a/pkg/settings/types.go b/pkg/settings/types.go index d139f5547..65d921d68 100644 --- a/pkg/settings/types.go +++ b/pkg/settings/types.go @@ -185,9 +185,22 @@ input: break } - // Hande changed or missing value + // Handle changed or missing value out = append(out, i) } return } + +// New returns all new values (that do not exist in the original set) +func (set ValueSet) New(in ValueSet) (out ValueSet) { + org := set.KV() + + for _, v := range in { + if !org.Has(v.Name) { + out = append(out, v) + } + } + + return +} diff --git a/system/importer/importer.go b/system/importer/importer.go index 0810dc510..d73ff984c 100644 --- a/system/importer/importer.go +++ b/system/importer/importer.go @@ -32,19 +32,32 @@ func NewImporter(p importer.PermissionImporter, s importer.SettingImporter, ri * } } +// Cast reads & translates input data into internal structures +// +// Handles base level for system resources (roles, permissions, settings) +// +// It skips all nil importers (see NewImporter() fn.) func (imp *Importer) Cast(in interface{}) (err error) { return deinterfacer.Each(in, func(index int, key string, val interface{}) (err error) { switch key { case "roles": - return imp.roles.CastSet(val) + if imp.roles != nil { + return imp.roles.CastSet(val) + } case "role": - return imp.roles.CastSet([]interface{}{val}) + if imp.roles != nil { + return imp.roles.CastSet([]interface{}{val}) + } case "allow", "deny": - return imp.permissions.CastResourcesSet(key, val) + if imp.permissions != nil { + return imp.permissions.CastResourcesSet(key, val) + } case "settings": - return imp.settings.CastSet(val) + if imp.settings != nil { + return imp.settings.CastSet(val) + } default: err = fmt.Errorf("unexpected key %q", key) @@ -54,6 +67,9 @@ func (imp *Importer) Cast(in interface{}) (err error) { }) } +// Store uses system services to store imported & casted data into storage +// +// It skips all nil importers (see NewImporter() fn.) func (imp *Importer) Store( ctx context.Context, rk roleKeeper, @@ -66,20 +82,26 @@ func (imp *Importer) Store( return } - // Make sure we properly replace role handles with IDs - roles.Walk(func(role *types.Role) error { - imp.permissions.UpdateRoles(role.Handle, role.ID) - return nil - }) - - err = imp.permissions.Store(ctx, pk) - if err != nil { - return + if imp.permissions != nil { + // Make sure we properly replace role handles with IDs + roles.Walk(func(role *types.Role) error { + imp.permissions.UpdateRoles(role.Handle, role.ID) + return nil + }) } - err = imp.settings.Store(ctx, sk) - if err != nil { - return errors.Wrap(err, "could not import settings") + if imp.permissions != nil { + err = imp.permissions.Store(ctx, pk) + if err != nil { + return + } + } + + if imp.settings != nil { + err = imp.settings.Store(ctx, sk) + if err != nil { + return errors.Wrap(err, "could not import settings") + } } return nil diff --git a/system/provision-config.go b/system/provision-config.go index 31f421d2d..5bd8c472c 100644 --- a/system/provision-config.go +++ b/system/provision-config.go @@ -2,16 +2,19 @@ package system import ( "context" + "io" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/titpetric/factory" "go.uber.org/zap" + "gopkg.in/yaml.v2" "github.com/cortezaproject/corteza-server/pkg/auth" "github.com/cortezaproject/corteza-server/pkg/cli" impAux "github.com/cortezaproject/corteza-server/pkg/importer" "github.com/cortezaproject/corteza-server/pkg/permissions" + "github.com/cortezaproject/corteza-server/pkg/settings" provision "github.com/cortezaproject/corteza-server/provision/system" "github.com/cortezaproject/corteza-server/system/importer" "github.com/cortezaproject/corteza-server/system/repository" @@ -19,18 +22,21 @@ import ( "github.com/cortezaproject/corteza-server/system/types" ) -func provisionConfig(ctx context.Context, cmd *cobra.Command, c *cli.Config) error { +func provisionConfig(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) { c.Log.Debug("running configuration provision") c.InitServices(ctx, c) + var provisioned bool + // Make sure we have all full access for provisioning ctx = auth.SetSuperUserContext(ctx) - if provisioned, err := isProvisioned(ctx); err != nil { + // if system is already provisioned, we do partial provisioning: + // missing settings only + if provisioned, err = isProvisioned(ctx); err != nil { return err } else if provisioned { c.Log.Debug("configuration already provisioned") - return nil } readers, err := impAux.ReadStatic(provision.Asset) @@ -38,6 +44,10 @@ func provisionConfig(ctx context.Context, cmd *cobra.Command, c *cli.Config) err return err } + if provisioned { + return partialImportSettings(ctx, service.DefaultSettings, readers...) + } + return errors.Wrap( importer.Import(ctx, readers...), "could not provision configuration for system service", @@ -100,3 +110,53 @@ func makeDefaultApplications(ctx context.Context, cmd *cobra.Command, c *cli.Con return nil }) } + +// Partial import of settings from provision files +func partialImportSettings(ctx context.Context, ss service.SettingsService, ff ...io.Reader) (err error) { + var ( + // decoded content from YAML files + aux interface{} + + si = settings.NewImporter() + + // importer w/o permissions & roles + // we need only settings + imp = importer.NewImporter(nil, si, nil) + + // current value + current settings.ValueSet + + // unexisting values + unex settings.ValueSet + ) + + for _, f := range ff { + if err = yaml.NewDecoder(f).Decode(&aux); err != nil { + return + } + + err = imp.Cast(aux) + if err != nil { + return + } + } + + ss = ss.With(ctx) + + // Get all "current" settings storage + current, err = ss.FindByPrefix("") + if err != nil { + return + } + + // Compare current settings with imported, get all that do not exist yet + if unex = si.GetValues(); len(unex) > 0 { + // Store non existing + err = ss.BulkSet(current.New(unex)) + if err != nil { + return + } + } + + return nil +} diff --git a/system/service/settings.go b/system/service/settings.go index aab5f2d5b..605cf0553 100644 --- a/system/service/settings.go +++ b/system/service/settings.go @@ -145,7 +145,11 @@ func (svc settings) AutoDiscovery() (err error) { } discovered, err = authSettingsAutoDiscovery(svc.logger, current) - if err != nil || len(discovered) == 0 { + if err != nil { + return + } + + if len(discovered) == 0 { return }