Add ability to provision (partial) unexisting settings

We moved a couple of strings from code to provision files and
we need to handle existing systems accordingly

pkg/settings/importer.go

@@ -54,3 +54,7 @@ func (imp *Importer) addSetting(name string, value interface{}) (err error) {
 func (imp *Importer) Store(ctx context.Context, k ImportKeeper) (err error) {
 	return k.BulkSet(imp.settings)
 }
+
+func (imp *Importer) GetValues() ValueSet {
+	return imp.settings
+}

pkg/settings/types.go

@@ -185,9 +185,22 @@ input:
 			break
 		}
 
-		// Hande changed or missing value
+		// Handle changed or missing value
 		out = append(out, i)
 	}
 
 	return
 }
+
+// New returns all new values (that do not exist in the original set)
+func (set ValueSet) New(in ValueSet) (out ValueSet) {
+	org := set.KV()
+
+	for _, v := range in {
+		if !org.Has(v.Name) {
+			out = append(out, v)
+		}
+	}
+
+	return
+}

system/importer/importer.go

@@ -32,19 +32,32 @@ func NewImporter(p importer.PermissionImporter, s importer.SettingImporter, ri *
 	}
 }
 
+// Cast reads & translates input data into internal structures
+//
+// Handles base level for system resources (roles, permissions, settings)
+//
+// It skips all nil importers (see NewImporter() fn.)
 func (imp *Importer) Cast(in interface{}) (err error) {
 	return deinterfacer.Each(in, func(index int, key string, val interface{}) (err error) {
 		switch key {
 		case "roles":
-			return imp.roles.CastSet(val)
+			if imp.roles != nil {
+				return imp.roles.CastSet(val)
+			}
 		case "role":
-			return imp.roles.CastSet([]interface{}{val})
+			if imp.roles != nil {
+				return imp.roles.CastSet([]interface{}{val})
+			}
 
 		case "allow", "deny":
-			return imp.permissions.CastResourcesSet(key, val)
+			if imp.permissions != nil {
+				return imp.permissions.CastResourcesSet(key, val)
+			}
 
 		case "settings":
-			return imp.settings.CastSet(val)
+			if imp.settings != nil {
+				return imp.settings.CastSet(val)
+			}
 
 		default:
 			err = fmt.Errorf("unexpected key %q", key)
@@ -54,6 +67,9 @@ func (imp *Importer) Cast(in interface{}) (err error) {
 	})
 }
 
+// Store uses system services to store imported & casted data into storage
+//
+// It skips all nil importers (see NewImporter() fn.)
 func (imp *Importer) Store(
 	ctx context.Context,
 	rk roleKeeper,
@@ -66,20 +82,26 @@ func (imp *Importer) Store(
 		return
 	}
 
-	// Make sure we properly replace role handles with IDs
-	roles.Walk(func(role *types.Role) error {
-		imp.permissions.UpdateRoles(role.Handle, role.ID)
-		return nil
-	})
-
-	err = imp.permissions.Store(ctx, pk)
-	if err != nil {
-		return
+	if imp.permissions != nil {
+		// Make sure we properly replace role handles with IDs
+		roles.Walk(func(role *types.Role) error {
+			imp.permissions.UpdateRoles(role.Handle, role.ID)
+			return nil
+		})
 	}
 
-	err = imp.settings.Store(ctx, sk)
-	if err != nil {
-		return errors.Wrap(err, "could not import settings")
+	if imp.permissions != nil {
+		err = imp.permissions.Store(ctx, pk)
+		if err != nil {
+			return
+		}
+	}
+
+	if imp.settings != nil {
+		err = imp.settings.Store(ctx, sk)
+		if err != nil {
+			return errors.Wrap(err, "could not import settings")
+		}
 	}
 
 	return nil

system/provision-config.go

@@ -2,16 +2,19 @@ package system
 
 import (
 	"context"
+	"io"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/titpetric/factory"
 	"go.uber.org/zap"
+	"gopkg.in/yaml.v2"
 
 	"github.com/cortezaproject/corteza-server/pkg/auth"
 	"github.com/cortezaproject/corteza-server/pkg/cli"
 	impAux "github.com/cortezaproject/corteza-server/pkg/importer"
 	"github.com/cortezaproject/corteza-server/pkg/permissions"
+	"github.com/cortezaproject/corteza-server/pkg/settings"
 	provision "github.com/cortezaproject/corteza-server/provision/system"
 	"github.com/cortezaproject/corteza-server/system/importer"
 	"github.com/cortezaproject/corteza-server/system/repository"
@@ -19,18 +22,21 @@ import (
 	"github.com/cortezaproject/corteza-server/system/types"
 )
 
-func provisionConfig(ctx context.Context, cmd *cobra.Command, c *cli.Config) error {
+func provisionConfig(ctx context.Context, cmd *cobra.Command, c *cli.Config) (err error) {
 	c.Log.Debug("running configuration provision")
 	c.InitServices(ctx, c)
 
+	var provisioned bool
+
 	// Make sure we have all full access for provisioning
 	ctx = auth.SetSuperUserContext(ctx)
 
-	if provisioned, err := isProvisioned(ctx); err != nil {
+	// if system is already provisioned, we do partial provisioning:
+	// missing settings only
+	if provisioned, err = isProvisioned(ctx); err != nil {
 		return err
 	} else if provisioned {
 		c.Log.Debug("configuration already provisioned")
-		return nil
 	}
 
 	readers, err := impAux.ReadStatic(provision.Asset)
@@ -38,6 +44,10 @@ func provisionConfig(ctx context.Context, cmd *cobra.Command, c *cli.Config) err
 		return err
 	}
 
+	if provisioned {
+		return partialImportSettings(ctx, service.DefaultSettings, readers...)
+	}
+
 	return errors.Wrap(
 		importer.Import(ctx, readers...),
 		"could not provision configuration for system service",
@@ -100,3 +110,53 @@ func makeDefaultApplications(ctx context.Context, cmd *cobra.Command, c *cli.Con
 		return nil
 	})
 }
+
+// Partial import of settings from provision files
+func partialImportSettings(ctx context.Context, ss service.SettingsService, ff ...io.Reader) (err error) {
+	var (
+		// decoded content from YAML files
+		aux interface{}
+
+		si = settings.NewImporter()
+
+		// importer w/o permissions & roles
+		// we need only settings
+		imp = importer.NewImporter(nil, si, nil)
+
+		// current value
+		current settings.ValueSet
+
+		// unexisting values
+		unex settings.ValueSet
+	)
+
+	for _, f := range ff {
+		if err = yaml.NewDecoder(f).Decode(&aux); err != nil {
+			return
+		}
+
+		err = imp.Cast(aux)
+		if err != nil {
+			return
+		}
+	}
+
+	ss = ss.With(ctx)
+
+	// Get all "current" settings storage
+	current, err = ss.FindByPrefix("")
+	if err != nil {
+		return
+	}
+
+	// Compare current settings with imported, get all that do not exist yet
+	if unex = si.GetValues(); len(unex) > 0 {
+		// Store non existing
+		err = ss.BulkSet(current.New(unex))
+		if err != nil {
+			return
+		}
+	}
+
+	return nil
+}

system/service/settings.go

@@ -145,7 +145,11 @@ func (svc settings) AutoDiscovery() (err error) {
 	}
 
 	discovered, err = authSettingsAutoDiscovery(svc.logger, current)
-	if err != nil || len(discovered) == 0 {
+	if err != nil {
+		return
+	}
+
+	if len(discovered) == 0 {
 		return
 	}