Rework provisioning with envoy

app/boot_levels.go

@@ -280,6 +280,9 @@ func (app *CortezaApp) Provision(ctx context.Context) (err error) {
 		if err != nil {
 			return err
 		}
+
+		// Provisioning doesn't automatically reload rbac rules, so this is required
+		rbac.Global().Reload(ctx)
 	}
 
 	app.lvl = bootLevelProvisioned

app/cli.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"github.com/cortezaproject/corteza-server/pkg/cli"
+	"github.com/cortezaproject/corteza-server/pkg/rbac"
 	systemCommands "github.com/cortezaproject/corteza-server/system/commands"
 )
 
@@ -33,6 +34,8 @@ func (app *CortezaApp) InitCLI() {
 			return
 		}
 
+		// Provisioning doesn't automatically reload rbac rules, so this is required
+		rbac.Global().Reload(ctx)
 		return
 	})
 

provision/compose/provision.go

@@ -2,35 +2,51 @@ package compose
 
 import (
 	"context"
-	"github.com/cortezaproject/corteza-server/compose/importer"
-	"github.com/cortezaproject/corteza-server/compose/types"
-	impAux "github.com/cortezaproject/corteza-server/pkg/importer"
+
+	"github.com/cortezaproject/corteza-server/provision/util"
 	"github.com/cortezaproject/corteza-server/store"
 	"go.uber.org/zap"
 )
 
-// provision only where there are no namespaces
-func hasNamespaces(ctx context.Context, s store.Storer) (bool, error) {
-	if set, _, err := store.SearchComposeNamespaces(ctx, s, types.NamespaceFilter{}); err != nil {
+func hasNamespaces(ctx context.Context, s store.Storer, slug string) (bool, error) {
+	ns, err := store.LookupComposeNamespaceBySlug(ctx, s, slug)
+	if err == store.ErrNotFound {
+		return false, nil
+	} else if err != nil {
 		return false, err
-	} else {
-		return len(set) > 0, nil
 	}
+	return ns != nil, nil
 }
 
 func Provision(ctx context.Context, log *zap.Logger, s store.Storer) error {
-	if namespacesExist, err := hasNamespaces(ctx, s); err != nil {
+	log.Info("provisioning compose: system")
+	if err := util.EncodeStatik(ctx, s, Asset, "/system"); err != nil {
 		return err
-	} else if !namespacesExist {
-		log.Info("provisioning compose")
-		readers, err := impAux.ReadStatic(Asset)
-		if err != nil {
+	}
+	log.Info("provisioned compose: system")
+
+	if has, err := hasNamespaces(ctx, s, "crm"); !has && err == nil {
+		log.Info("provisioning compose: crm")
+		if err := util.EncodeStatik(ctx, s, Asset, "/crm"); err != nil {
 			return err
 		}
-
-		return importer.Import(ctx, nil, readers...)
+		log.Info("provisioned compose: crm")
+	} else if err != nil {
+		return err
 	} else {
-		log.Info("compose already provisioned")
+		log.Info("[skip] provisioning compose: crm")
+	}
+
+	if has, err := hasNamespaces(ctx, s, "service-solution"); !has && err == nil {
+		log.Info("provisioning compose: service-solution")
+		if err := util.EncodeStatik(ctx, s, Asset, "/service-solution"); err != nil {
+			return err
+		}
+		log.Info("provisioned compose: service-solution")
+	} else if err != nil {
+		return err
+	} else {
+		log.Info("[skip] provisioning compose: service-solution")
 	}
 
 	return nil

provision/compose/src/crm/1100_modules.yaml

@@ -8769,78 +8769,6 @@ modules:
         options:
           format: "0000000"
           precision: 0
-      ScoreBudget:
-        label: Budget
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreAuthority:
-        label: Authority
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreNeed:
-        label: Need
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreTimescale:
-        label: Timescale
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreTurnover:
-        label: Turnover
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreYearsInBusiness:
-        label: Yrs in Business
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreSource:
-        label: Source
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreProductKnowlege:
-        label: Product Knowlege
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreTechnicalDepth:
-        label: Technical Depth
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreCompetition:
-        label: Competition
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreCriticality:
-        label: Criticality
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
-      ScoreRepeatPurchase:
-        label: Repeat Purchase
-        kind: Number
-        options:
-          format: "0"
-          precision: 2
   Solution:
     name: Solution
     fields:

provision/compose/src/crm/1400_pages.yaml

@@ -3920,39 +3920,6 @@ pages:
                 variants: {bodyBg: white, border: primary, headerBg: white, headerText: primary}
               kind: Record
               xywh: [9, 0, 3, 12]
-            - title: Opportunity Qualification Weights
-              options:
-                fields:
-                - name: ScoreBudget
-                - name: ScoreAuthority
-                - name: ScoreNeed
-                - name: ScoreTimescale
-              style:
-                variants: {bodyBg: white, border: primary, headerBg: white, headerText: primary}
-              kind: Record
-              xywh: [0, 12, 3, 8]
-            - title: Opportunity Facts Weighting
-              options:
-                fields:
-                - name: ScoreTurnover
-                - name: ScoreYearsInBusiness
-                - name: ScoreSource
-                - name: ScoreProductKnowlege
-              style:
-                variants: {bodyBg: white, border: primary, headerBg: white, headerText: primary}
-              kind: Record
-              xywh: [3, 12, 3, 8]
-            - title: Opportunity Cutsom Weighting
-              options:
-                fields:
-                - name: ScoreTechnicalDepth
-                - name: ScoreCompetition
-                - name: ScoreCriticality
-                - name: ScoreRepeatPurchase
-              style:
-                variants: {bodyBg: white, border: primary, headerBg: white, headerText: primary}
-              kind: Record
-              xywh: [6, 12, 3, 8]
             visible: false
         visible: true
     visible: true

provision/compose/src/service-solution/1100_modules.yaml

@@ -695,14 +695,14 @@ modules:
         label: Total time of all cases
         kind: Number
         options:
-          format: ""
+          format: "0.00"
           precision: "2"
           suffix: ' hour(s)'
       TotalCost:
         label: Total cost of all cases
         kind: Number
         options:
-          format: ""
+          format: "0.00"
           precision: "2"
           prefix: $
   Attachement:
@@ -856,14 +856,14 @@ modules:
         label: Total Time
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 2
           suffix: ' hour(s)'
       TotalCost:
         label: Total Cost
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 2
           prefix: $
       MailingList:
@@ -1600,21 +1600,21 @@ modules:
         label: Cost per hour
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 2
           prefix: $
       TotalTime:
         label: Total time of all cases
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 0
           suffix: ' hour(s)'
       TotalCost:
         label: Total cost of all cases
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: "2"
           prefix: $
       DepartmentSelect:
@@ -1939,14 +1939,14 @@ modules:
         label: Total time of all cases
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: "2"
           suffix: ' hour(s)'
       TotalCost:
         label: Total cost of all cases
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 2
           prefix: $
       ProductSelect:
@@ -1970,8 +1970,9 @@ modules:
         label: Default cost per hour
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 0
+          prefix: $
       DefaultDepartment:
         label: Default department for new ticket
         kind: Record
@@ -1985,7 +1986,7 @@ modules:
         label: Default time for a case update
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 2
           suffix: ' hour(s)'
       DefaultSupportChannel:
@@ -2067,14 +2068,14 @@ modules:
         label: Time spend
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 2
           suffix: ' hour(s)'
       Cost:
         label: Cost
         kind: Number
         options:
-          format: 0
+          format: "0.00"
           precision: 2
           prefix: $
       Department:

provision/system/provision.go

@@ -2,18 +2,14 @@ package system
 
 import (
 	"context"
-	"github.com/cortezaproject/corteza-server/pkg/id"
-	impAux "github.com/cortezaproject/corteza-server/pkg/importer"
+	"time"
+
 	"github.com/cortezaproject/corteza-server/pkg/rbac"
-	"github.com/cortezaproject/corteza-server/pkg/settings"
+	"github.com/cortezaproject/corteza-server/provision/util"
 	"github.com/cortezaproject/corteza-server/store"
-	"github.com/cortezaproject/corteza-server/system/importer"
 	"github.com/cortezaproject/corteza-server/system/service"
 	"github.com/cortezaproject/corteza-server/system/types"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
-	"io"
-	"time"
 )
 
 type (
@@ -23,80 +19,40 @@ type (
 	}
 )
 
-// Check if any roels
-func checkRoles(ctx context.Context, s store.Storer) (bool, error) {
+func provisionRoles(ctx context.Context, s store.Storer) error {
 	if set, _, err := store.SearchRoles(ctx, s, types.RoleFilter{}); err != nil {
-		return false, err
-	} else {
-		return len(set) > 0, nil
+		return err
+	} else if len(set) > 0 {
+		return nil
 	}
-}
 
-// Check if any RBAC rules exist
-func checkRbacRules(ctx context.Context, s store.Storer) (bool, error) {
-	if set, _, err := store.SearchRbacRules(ctx, s, rbac.RuleFilter{}); err != nil {
-		return false, err
-	} else {
-		return len(set) > 0, nil
+	rr := types.RoleSet{
+		&types.Role{ID: rbac.AdminsRoleID, Name: "Administrators", Handle: "admins"},
+		&types.Role{ID: rbac.EveryoneRoleID, Name: "Everyone", Handle: "everyone"},
 	}
+
+	err := rr.Walk(func(r *types.Role) error {
+		r.CreatedAt = time.Now()
+		return store.CreateRole(ctx, s, r)
+	})
+
+	return err
 }
 
 func Provision(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
-	var (
-		hasRoles, hasRbacRules bool
-		readers                []io.Reader
-	)
-
-	if hasRoles, err = checkRoles(ctx, s); err != nil {
-		return err
-	} else if !hasRoles {
-		rr := types.RoleSet{
-			&types.Role{ID: rbac.AdminsRoleID, Name: "Administrators", Handle: "admins"},
-			&types.Role{ID: rbac.EveryoneRoleID, Name: "Everyone", Handle: "everyone"},
-		}
-
-		err = rr.Walk(func(r *types.Role) error {
-			r.CreatedAt = time.Now()
-			return store.CreateRole(ctx, s, r)
-		})
-
-		if err != nil {
-			return err
-		}
-	}
-
-	if hasRbacRules, err = checkRbacRules(ctx, s); err != nil {
+	log.Info("provisioning system")
+	err = provisionRoles(ctx, s)
+	if err != nil {
 		return err
 	}
 
-	if !hasRbacRules {
-		log.Info("provisioning system")
-		readers, err = impAux.ReadStatic(Asset)
-		if err != nil {
-			return err
-		}
-
-		if err = importer.Import(ctx, readers...); err != nil {
-			return err
-		}
-
-	} else {
-		log.Info("provisioning system settings")
-		// When already provisioned, make sure settings are re-provisioned
-		readers, err = impAux.ReadStatic(Asset)
-		if err != nil {
-			return err
-		}
-
-		if err = partialImportSettings(ctx, service.DefaultSettings, readers...); err != nil {
-			return err
-		}
-	}
-
-	if err = makeDefaultApplications(ctx, log, s); err != nil {
-		return
-	}
+	// Provision from yumles
+	// - access control
+	// - settings
+	// - applications
+	err = util.EncodeStatik(ctx, s, Asset, "/")
 
+	// These ones need some extra things, so we'll leave them there
 	if err = authSettingsAutoDiscovery(ctx, log, service.DefaultSettings); err != nil {
 		return
 	}
@@ -113,205 +69,6 @@ func Provision(ctx context.Context, log *zap.Logger, s store.Storer) (err error)
 		return
 	}
 
-	return nil
-}
-
-// Updates default application directly in the store
-func makeDefaultApplications(ctx context.Context, log *zap.Logger, s store.Storer) error {
-	var (
-		now        = time.Now()
-		aa, _, err = s.SearchApplications(ctx, types.ApplicationFilter{})
-	)
-	if err != nil {
-		return err
-	}
-
-	// Update icon & logo on all apps
-	const (
-		oldIconUrl = "/applications/crust_favicon.png"
-		oldLogoUrl = "/applications/crust.jpg"
-
-		newIconUrl = "/applications/default_icon.png"
-		newLogoUrl = "/applications/default_logo.jpg"
-	)
-
-	for a := 0; a < len(aa); a++ {
-		var dirty bool
-
-		if aa[a].Unify == nil {
-			continue
-		}
-
-		if aa[a].Unify.Icon == oldIconUrl {
-			aa[a].Unify.Icon = newIconUrl
-			dirty = true
-		}
-
-		if aa[a].Unify.Logo == oldLogoUrl {
-			aa[a].Unify.Logo = newLogoUrl
-			dirty = true
-		}
-
-		if !dirty {
-			continue
-		}
-
-		aa[a].UpdatedAt = &now
-
-		if err = s.UpdateApplication(ctx, aa[a]); err != nil {
-			return err
-		}
-	}
-
-	// List of apps to create.
-	//
-	// We use Unify.Url field for matching,
-	// so make sure it's always present!
-	defApps := types.ApplicationSet{
-		&types.Application{
-			Name:    "Messaging",
-			Enabled: true,
-			Unify: &types.ApplicationUnify{
-				Listed: true,
-				Icon:   newIconUrl,
-				Logo:   newLogoUrl,
-				Url:    "/messaging",
-			},
-		},
-
-		&types.Application{
-			Name:    "Low Code",
-			Enabled: true,
-			Unify: &types.ApplicationUnify{
-				Listed: true,
-				Icon:   newIconUrl,
-				Logo:   newLogoUrl,
-				Url:    "/admin",
-			},
-		},
-
-		&types.Application{
-			Name:    "Corteza Admin Area",
-			Enabled: true,
-			Unify: &types.ApplicationUnify{
-				Listed: true,
-				Icon:   newIconUrl,
-				Logo:   newLogoUrl,
-				Url:    "/admin",
-			},
-		},
-
-		&types.Application{
-			Name:    "Corteza Jitsi Bridge",
-			Enabled: true,
-			Unify: &types.ApplicationUnify{
-				Listed: true,
-				Icon:   "/applications/jitsi_icon.png",
-				Logo:   "/applications/jitsi.png",
-				Url:    "/bridge/jitsi/",
-			},
-		},
-
-		&types.Application{
-			Name:    "Google Maps",
-			Enabled: true,
-			Unify: &types.ApplicationUnify{
-				Listed: true,
-				Icon:   "/applications/google_maps_icon.png",
-				Logo:   "/applications/google_maps.png",
-				Url:    "/bridge/google-maps/",
-			},
-		},
-
-		&types.Application{
-			Name:    "CRM",
-			Enabled: true,
-			Unify: &types.ApplicationUnify{
-				Listed: true,
-				Icon:   newIconUrl,
-				Logo:   newLogoUrl,
-				Url:    "/compose/ns/crm/pages",
-			},
-		},
-
-		&types.Application{
-			Name:    "Service Solution",
-			Enabled: true,
-			Unify: &types.ApplicationUnify{
-				Listed: true,
-				Icon:   newIconUrl,
-				Logo:   newLogoUrl,
-				Url:    "/compose/ns/service-solution/pages",
-			},
-		},
-	}
-
-	return defApps.Walk(func(defApp *types.Application) error {
-		for _, a := range aa {
-			if a.Unify != nil && a.Unify.Url == defApp.Unify.Url {
-				// App already added.
-				return nil
-			}
-		}
-
-		defApp.ID = id.Next()
-		defApp.CreatedAt = time.Now()
-
-		err = s.CreateApplication(ctx, defApp)
-		log.Info(
-			"creating default application",
-			zap.String("name", defApp.Name),
-			zap.Uint64("name", defApp.ID),
-			zap.Error(err),
-		)
-		return err
-	})
-}
-
-// Partial import of settings from provision files
-func partialImportSettings(ctx context.Context, ss settingsService, ff ...io.Reader) (err error) {
-	var (
-		// decoded content from YAML files
-		aux interface{}
-
-		si = settings.NewImporter()
-
-		// importer w/o permissions & roles
-		// we need only settings
-		imp = importer.NewImporter(nil, si, nil)
-
-		// current value
-		current types.SettingValueSet
-
-		// unexisting values
-		unex types.SettingValueSet
-	)
-
-	for _, f := range ff {
-		if err = yaml.NewDecoder(f).Decode(&aux); err != nil {
-			return
-		}
-
-		err = imp.Cast(aux)
-		if err != nil {
-			return
-		}
-	}
-
-	// Get all "current" settings storage
-	current, err = ss.FindByPrefix(ctx)
-	if err != nil {
-		return
-	}
-
-	// Compare current settings with imported, get all that do not exist yet
-	if unex = si.GetValues(); len(unex) > 0 {
-		// Store non existing
-		err = ss.BulkSet(ctx, current.New(unex))
-		if err != nil {
-			return
-		}
-	}
-
+	log.Info("provisioned system")
 	return nil
 }

provision/system/src/0200_applications.yaml

@@ -0,0 +1,57 @@
+applications:
+  - name: Messaging
+    enabled: true
+    unify:
+      listed: true
+      icon: /applications/default_icon.png
+      logo: /applications/default_logo.jpg
+      url: /messaging
+
+  - name: Low Code
+    enabled: true
+    unify:
+      listed: true
+      icon: /applications/default_icon.png
+      logo: /applications/default_logo.jpg
+      url: /admin
+
+
+  - name: Corteza Admin Area
+    enabled: true
+    unify:
+      listed: true
+      icon: /applications/default_icon.png
+      logo: /applications/default_logo.jpg
+      url: /admin
+
+  - name: Corteza Jitsi Bridge
+    enabled: true
+    unify:
+      listed: true
+      icon: /applications/jitsi_icon.png
+      logo: /applications/jitsi.png
+      url: /bridge/jitsi/
+
+  - name: Google Maps
+    enabled: true
+    unify:
+      listed: true
+      icon: /applications/google_maps_icon.png
+      logo: /applications/google_maps.png
+      url: /bridge/google-maps/
+
+  - name: CRM
+    enabled: true
+    unify:
+      listed: true
+      icon: newIconUrl
+      logo: newLogoUrl
+      url: /compose/ns/crm/pages
+
+  - name: Service Solution
+    enabled: true
+    unify:
+      listed: true
+      icon: newIconUrl
+      logo: newLogoUrl
+      url: /compose/ns/service-solution/pages

provision/update.sh

@@ -20,7 +20,8 @@ function download {
 
 function copyExtConfig {
   echo -e "\033[32mCopying ${2} ${1}\033[39m ... "
-  cp "${DIR}/${1}/config/${2}.yaml" "./compose/src/${2}_${1}.yaml"
+  mkdir -p "./compose/src/${1}"
+  cp "${DIR}/${1}/config/${2}.yaml" "./compose/src/${1}/${2}.yaml"
 }
 
 function cleanup {

provision/util/decode_statik.go

@@ -0,0 +1,95 @@
+package util
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+
+	"github.com/cortezaproject/corteza-server/pkg/envoy"
+	"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
+	es "github.com/cortezaproject/corteza-server/pkg/envoy/store"
+	"github.com/cortezaproject/corteza-server/pkg/envoy/yaml"
+	"github.com/cortezaproject/corteza-server/store"
+	"github.com/goware/statik/fs"
+)
+
+type (
+	decoder interface {
+		CanDecodeExt(string) bool
+		CanDecodeFile(io.Reader) bool
+		Decode(context.Context, io.Reader, *envoy.DecoderOpts) ([]resource.Interface, error)
+	}
+)
+
+func StatikResources(ctx context.Context, d decoder, sfs http.FileSystem, root string) ([]resource.Interface, error) {
+	nn := make(resource.InterfaceSet, 0, 100)
+	err := fs.Walk(sfs, root, func(p string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		f, err := sfs.Open(p)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		if !d.CanDecodeFile(f) && !d.CanDecodeExt(info.Name()) {
+			return nil
+		}
+
+		if d == nil {
+			// no decoder found
+			return nil
+		}
+
+		_, err = f.Seek(0, 0)
+		if err != nil {
+			return err
+		}
+
+		dir, fn := path.Split(p)
+		do := &envoy.DecoderOpts{
+			Name: fn,
+			Path: dir,
+		}
+
+		if dnn, err := d.Decode(ctx, f, do); err != nil {
+			return fmt.Errorf("failed to decode %s: %w", info.Name(), err)
+		} else {
+			nn = append(nn, dnn...)
+		}
+
+		return nil
+	})
+
+	return nn, err
+}
+
+func EncodeStatik(ctx context.Context, s store.Storer, rfs string, root string) error {
+	sfs, err := fs.New(rfs)
+	if err != nil {
+		return err
+	}
+
+	yd := yaml.Decoder()
+	se := es.NewStoreEncoder(s, &es.EncoderConfig{
+		OnExisting: es.MergeLeft,
+	})
+
+	nn, err := StatikResources(ctx, yd, sfs, root)
+	if err != nil {
+		return err
+	}
+
+	bld := envoy.NewBuilder(se)
+	g, err := bld.Build(ctx, nn...)
+	if err != nil {
+		return err
+	}
+
+	return envoy.Encode(ctx, g, se)
+}