DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Refactored provisioning

Browse Source
This commit is contained in:
Denis Arh
2020-08-24 15:38:42 +02:00
parent 9013923e4b
commit 2c1368262b
17 changed files with 153 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose/provision.go
-101
View File
@@ -1,101 +0,0 @@
package compose
import (
"context"
"io"
"github.com/pkg/errors"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/compose/importer"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
"github.com/cortezaproject/corteza-server/pkg/auth"
impAux "github.com/cortezaproject/corteza-server/pkg/importer"
"github.com/cortezaproject/corteza-server/pkg/settings"
provision "github.com/cortezaproject/corteza-server/provision/compose"
)
func provisionConfig(ctx context.Context, log *zap.Logger) (err error) {
log.Debug("running configuration provision")
var provisioned bool
// Make sure we have all full access for provisioning
ctx = auth.SetSuperUserContext(ctx)
if provisioned, err = isProvisioned(ctx); err != nil {
return err
} else if provisioned {
log.Debug("configuration already provisioned")
}
readers, err := impAux.ReadStatic(provision.Asset)
if err != nil {
return err
}
if provisioned {
return partialImportSettings(ctx, service.DefaultSettings, readers...)
}
return errors.Wrap(
importer.Import(ctx, nil, readers...),
"could not provision configuration for compose service",
)
}
// Provision only where there are no namespaces
func isProvisioned(ctx context.Context) (bool, error) {
_, f, err := service.DefaultNamespace.With(ctx).Find(types.NamespaceFilter{})
return f.Count > 0, err
}
// Partial import of settings from provision files
func partialImportSettings(ctx context.Context, ss settings.Service, ff ...io.Reader) (err error) {
var (
// decoded content from YAML files
aux interface{}
si = settings.NewImporter()
// importer w/o permissions & roles
// we need only settings
imp = importer.NewImporter(nil, nil, nil, nil, nil, si)
// current value
current settings.ValueSet
// unexisting values
unex settings.ValueSet
)
for _, f := range ff {
if err = yaml.NewDecoder(f).Decode(&aux); err != nil {
return
}
err = imp.Cast(aux)
if err != nil {
return
}
}
// Get all "current" settings storage
current, err = ss.FindByPrefix(ctx)
if err != nil {
return
}
// Compare current settings with imported, get all that do not exist yet
if unex = si.GetValues(); len(unex) > 0 {
// Store non existing
err = ss.BulkSet(ctx, current.New(unex))
if err != nil {
return
}
}
return nil
}
messaging/provision.go
-101
View File
@@ -1,101 +0,0 @@
package messaging
import (
"context"
"io"
"github.com/pkg/errors"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/messaging/importer"
"github.com/cortezaproject/corteza-server/messaging/service"
"github.com/cortezaproject/corteza-server/messaging/types"
"github.com/cortezaproject/corteza-server/pkg/auth"
impAux "github.com/cortezaproject/corteza-server/pkg/importer"
"github.com/cortezaproject/corteza-server/pkg/settings"
provision "github.com/cortezaproject/corteza-server/provision/messaging"
)
func provisionConfig(ctx context.Context, log *zap.Logger) (err error) {
log.Debug("running configuration provision")
var provisioned bool
// Make sure we have all full access for provisioning
ctx = auth.SetSuperUserContext(ctx)
if provisioned, err = isProvisioned(ctx); err != nil {
return err
} else if provisioned {
log.Debug("configuration already provisioned")
}
readers, err := impAux.ReadStatic(provision.Asset)
if err != nil {
return err
}
if provisioned {
return partialImportSettings(ctx, service.DefaultSettings, readers...)
}
return errors.Wrap(
importer.Import(ctx, readers...),
"could not provision configuration for messaging service",
)
}
// Provision ONLY when there are no channels (even if we find delete channels we abort provisioning
func isProvisioned(ctx context.Context) (bool, error) {
cc, _, err := service.DefaultChannel.With(ctx).Find(types.ChannelFilter{IncludeDeleted: true})
return len(cc) > 0, err
}
// Partial import of settings from provision files
func partialImportSettings(ctx context.Context, ss settings.Service, ff ...io.Reader) (err error) {
var (
// decoded content from YAML files
aux interface{}
si = settings.NewImporter()
// importer w/o permissions & roles
// we need only settings
imp = importer.NewImporter(nil, si, nil)
// current value
current settings.ValueSet
// unexisting values
unex settings.ValueSet
)
for _, f := range ff {
if err = yaml.NewDecoder(f).Decode(&aux); err != nil {
return
}
err = imp.Cast(aux)
if err != nil {
return
}
}
// Get all "current" settings storage
current, err = ss.FindByPrefix(ctx)
if err != nil {
return
}
// Compare current settings with imported, get all that do not exist yet
if unex = si.GetValues(); len(unex) > 0 {
// Store non existing
err = ss.BulkSet(ctx, current.New(unex))
if err != nil {
return
}
}
return nil
}
pkg/app/options/provision.go
-17
View File
@@ -1,17 +0,0 @@
package options
type (
ProvisionOpt struct {
Always bool `env:"PROVISION_ALWAYS"`
}
)
func Provision(pfix string) (o *ProvisionOpt) {
o = &ProvisionOpt{
Always: true,
}
fill(o, pfix)
return
}
provision/compose/placeholder.go
-1
View File
@@ -1 +0,0 @@
package compose
provision/compose/provision.go
+32
View File
@@ -0,0 +1,32 @@
package compose
import (
"context"
"github.com/cortezaproject/corteza-server/compose/importer"
"github.com/cortezaproject/corteza-server/compose/service"
"github.com/cortezaproject/corteza-server/compose/types"
impAux "github.com/cortezaproject/corteza-server/pkg/importer"
"go.uber.org/zap"
)
func Provision(ctx context.Context, log *zap.Logger) error {
if provisioned, err := notProvisioned(ctx); err != nil {
return err
} else if !provisioned {
log.Info("provisioning compose")
readers, err := impAux.ReadStatic(Asset)
if err != nil {
return err
}
return importer.Import(ctx, nil, readers...)
}
return nil
}
// provision only where there are no namespaces
func notProvisioned(ctx context.Context) (bool, error) {
_, f, err := service.DefaultNamespace.With(ctx).Find(types.NamespaceFilter{})
return f.Count == 0, err
}
provision/compose/src/0100_settings.yaml
-8
View File
@@ -1,8 +0,0 @@
settings:
ui.namespace-switcher.enabled: false
record.attachments.max-size: 10
record.attachments.mimetypes: []
page.attachments.max-size: 10
page.attachments.mimetypes: []
provision/compose/static.go
+1 -1
View File
File diff suppressed because one or more lines are too long
provision/messaging/placeholder.go
-1
View File
@@ -1 +0,0 @@
package messaging
provision/messaging/provision.go
+32
View File
@@ -0,0 +1,32 @@
package messaging
import (
"context"
"github.com/cortezaproject/corteza-server/messaging/importer"
"github.com/cortezaproject/corteza-server/messaging/service"
"github.com/cortezaproject/corteza-server/messaging/types"
impAux "github.com/cortezaproject/corteza-server/pkg/importer"
"go.uber.org/zap"
)
func Provision(ctx context.Context, log *zap.Logger) error {
if provisioned, err := notProvisioned(ctx); err != nil {
return err
} else if !provisioned {
log.Info("provisioning messaging")
readers, err := impAux.ReadStatic(Asset)
if err != nil {
return err
}
return importer.Import(ctx, readers...)
}
return nil
}
// Provision only where there are no channels
func notProvisioned(ctx context.Context) (bool, error) {
cc, _, err := service.DefaultChannel.With(ctx).Find(types.ChannelFilter{IncludeDeleted: true})
return len(cc) == 0, err
}
provision/messaging/src/0100_settings.yaml
-10
View File
@@ -1,10 +0,0 @@
settings:
ui.emoji.enabled: true
ui.browser-notifications.enabled: true
ui.browser-notifications.header: ${user} in ${channel}
ui.browser-notifications.message-trim: 200
message.attachments.enabled: true
message.attachments.max-size: 10
message.attachments.mimetypes: []
message.attachments.source.gallery.enabled: true
message.attachments.source.camera.enabled: true
provision/messaging/static.go
+1 -1
View File
@@ -3,4 +3,4 @@
// Package contains static assets.
package messaging
var Asset = "PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\x00 \x000000_access_control.yamlUT\x05\x00\x01\x80Cm8allow:\n everyone:\n messaging:\n - access\n\n admins:\n messaging:\n - access\n - grant\n - settings.read\n - settings.manage\n - channel.public.create\n - channel.private.create\n - channel.group.create\n\n messaging:channel:\n - update\n - leave\n - read\n - join\n - delete\n - undelete\n - archive\n - unarchive\n - members.manage\n - attachments.manage\n - message.attach\n - message.update.all\n - message.update.own\n - message.delete.all\n - message.delete.own\n - message.embed\n - message.send\n - message.reply\n - message.react\n\nPK\x07\x08\xae\x83\x83d\x8e\x02\x00\x00\x8e\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x12\x00 \x000100_settings.yamlUT\x05\x00\x01\x80Cm8settings:\n ui.emoji.enabled: true\n ui.browser-notifications.enabled: true\n ui.browser-notifications.header: ${user} in ${channel}\n ui.browser-notifications.message-trim: 200\n message.attachments.enabled: true\n message.attachments.max-size: 10\n message.attachments.mimetypes: []\n message.attachments.source.gallery.enabled: true\n message.attachments.source.camera.enabled: true\nPK\x07\x08Cy\xf0y\x82\x01\x00\x00\x82\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x12\x00 \x001000_channels.yamlUT\x05\x00\x01\x80Cm8channels:\n - name: General\n type: public\n - name: Random\n type: public\nPK\x07\x08\xe8\x83F\xf8O\x00\x00\x00O\x00\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x00\x00!(\xae\x83\x83d\x8e\x02\x00\x00\x8e\x02\x00\x00\x18\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x000000_access_control.yamlUT\x05\x00\x01\x80Cm8PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x00\x00!(Cy\xf0y\x82\x01\x00\x00\x82\x01\x00\x00\x12\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xdd\x02\x00\x000100_settings.yamlUT\x05\x00\x01\x80Cm8PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x00\x00!(\xe8\x83F\xf8O\x00\x00\x00O\x00\x00\x00\x12\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xa8\x04\x00\x001000_channels.yamlUT\x05\x00\x01\x80Cm8PK\x05\x06\x00\x00\x00\x00\x03\x00\x03\x00\xe1\x00\x00\x00@\x05\x00\x00\x00\x00"
var Asset = "PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\x00 \x000000_access_control.yamlUT\x05\x00\x01\x80Cm8allow:\n everyone:\n messaging:\n - access\n\n admins:\n messaging:\n - access\n - grant\n - settings.read\n - settings.manage\n - channel.public.create\n - channel.private.create\n - channel.group.create\n\n messaging:channel:\n - update\n - leave\n - read\n - join\n - delete\n - undelete\n - archive\n - unarchive\n - members.manage\n - attachments.manage\n - message.attach\n - message.update.all\n - message.update.own\n - message.delete.all\n - message.delete.own\n - message.embed\n - message.send\n - message.reply\n - message.react\n\nPK\x07\x08\xae\x83\x83d\x8e\x02\x00\x00\x8e\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x12\x00 \x001000_channels.yamlUT\x05\x00\x01\x80Cm8channels:\n - name: General\n type: public\n - name: Random\n type: public\nPK\x07\x08\xe8\x83F\xf8O\x00\x00\x00O\x00\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x00\x00!(\xae\x83\x83d\x8e\x02\x00\x00\x8e\x02\x00\x00\x18\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x000000_access_control.yamlUT\x05\x00\x01\x80Cm8PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x00\x00!(\xe8\x83F\xf8O\x00\x00\x00O\x00\x00\x00\x12\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xdd\x02\x00\x001000_channels.yamlUT\x05\x00\x01\x80Cm8PK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00\x98\x00\x00\x00u\x03\x00\x00\x00\x00"
system/provision_settings.go → provision/system/auth_autodiscovery.go
+5 -5
View File
@@ -3,6 +3,7 @@ package system
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/system/types"
"net/url"
"os"
"strings"
@@ -12,7 +13,6 @@ import (
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/pkg/rand"
settings "github.com/cortezaproject/corteza-server/pkg/settings"
"github.com/cortezaproject/corteza-server/system/service"
)
@@ -23,14 +23,14 @@ var (
// Discovers "auth.%" settings from the environment
//
// when other kinds of auto-discoverable settings come, lambdas inside will probably need a bit of refactoring
func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc settings.Service) (err error) {
func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc settingsService) (err error) {
type (
stringWrapper func() string
boolWrapper func() bool
)
var (
current settings.ValueSet
current types.SettingValueSet
)
if log == nil {
@@ -68,11 +68,11 @@ func authSettingsAutoDiscovery(ctx context.Context, log *zap.Logger, svc setting
if v != nil {
// Nothing to discover, already set
log.Info("already set", logger.MaskIf("value", v, maskSensitive))
log.Debug("already set", logger.MaskIf("value", v, maskSensitive))
return
}
v = &settings.Value{Name: name}
v = &types.SettingValue{Name: name}
value, envExists := os.LookupEnv(env)
system/provision_oidc.go → provision/system/oidc.go
+8 -10
View File
@@ -2,22 +2,20 @@ package system
import (
"context"
"strings"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/pkg/app/options"
"fmt"
"github.com/cortezaproject/corteza-server/pkg/auth"
"github.com/cortezaproject/corteza-server/pkg/options"
"github.com/cortezaproject/corteza-server/system/auth/external"
"github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
"strings"
)
// Provisions OIDC providers from PROVISION_OIDC_PROVIDER env variable
//
// Env variable should contains space delimited pairs of providers (<name> <provider> ....)
func oidcAutoDiscovery(ctx context.Context, log *zap.Logger) (err error) {
var provider = strings.TrimSpace(options.EnvString("", "PROVISION_OIDC_PROVIDER", ""))
var provider = strings.TrimSpace(options.EnvString("PROVISION_OIDC_PROVIDER", ""))
log.Debug("OIDC auto discovery provision",
zap.String("envkey", "PROVISION_OIDC_PROVIDER"),
@@ -36,7 +34,7 @@ func oidcAutoDiscovery(ctx context.Context, log *zap.Logger) (err error) {
)
if plen%2 == 1 {
return errors.New("expecting even number of providers")
return fmt.Errorf("expecting even number of providers")
}
for p := 0; p < plen; p = p + 2 {
@@ -73,7 +71,7 @@ func oidcAutoDiscovery(ctx context.Context, log *zap.Logger) (err error) {
return
}
func authAddExternals(ctx context.Context) (err error) {
func authAddExternals(ctx context.Context, log *zap.Logger) (err error) {
var (
kinds = []string{
"github",
@@ -93,7 +91,7 @@ func authAddExternals(ctx context.Context) (err error) {
for _, kind := range kinds {
env = "PROVISION_SETTINGS_AUTH_EXTERNAL_" + strings.ToUpper(kind)
p = strings.TrimSpace(options.EnvString("", env, ""))
p = strings.TrimSpace(options.EnvString(env, ""))
if len(p) == 0 {
continue
}
provision/system/placeholder.go
-1
View File
@@ -1 +0,0 @@
package system
system/provision.go → provision/system/provision.go
+56 -36
View File
@@ -2,58 +2,78 @@ package system
import (
"context"
"io"
"github.com/pkg/errors"
"github.com/titpetric/factory"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
"github.com/cortezaproject/corteza-server/pkg/auth"
impAux "github.com/cortezaproject/corteza-server/pkg/importer"
"github.com/cortezaproject/corteza-server/pkg/permissions"
"github.com/cortezaproject/corteza-server/pkg/settings"
provision "github.com/cortezaproject/corteza-server/provision/system"
"github.com/cortezaproject/corteza-server/system/importer"
"github.com/cortezaproject/corteza-server/system/repository"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/titpetric/factory"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
"io"
)
func provisionConfig(ctx context.Context, log *zap.Logger) (err error) {
log.Debug("running configuration provision")
type (
settingsService interface {
FindByPrefix(context.Context, ...string) (types.SettingValueSet, error)
BulkSet(context.Context, types.SettingValueSet) error
}
)
func Provision(ctx context.Context, log *zap.Logger) (err error) {
var provisioned bool
var readers []io.Reader
// Make sure we have all full access for provisioning
ctx = auth.SetSuperUserContext(ctx)
// if system is already provisioned, we do partial provisioning:
// missing settings only
if provisioned, err = isProvisioned(ctx); err != nil {
if provisioned, err = notProvisioned(ctx); err != nil {
return err
} else if provisioned {
log.Debug("configuration already provisioned")
} else if !provisioned {
log.Info("provisioning system")
readers, err = impAux.ReadStatic(Asset)
if err != nil {
return err
}
if err = importer.Import(ctx, readers...); err != nil {
return err
}
} else {
log.Info("provisioning system")
// When already provisioned, make sure settings are re-provisioned
readers, err = impAux.ReadStatic(Asset)
if err != nil {
return err
}
if err = partialImportSettings(ctx, service.DefaultSettings, readers...); err != nil {
return err
}
}
readers, err := impAux.ReadStatic(provision.Asset)
if err != nil {
return err
if err = makeDefaultApplications(ctx, log); err != nil {
return
}
if err = authSettingsAutoDiscovery(ctx, log, service.DefaultSettings); err != nil {
return
}
if err = authAddExternals(ctx, log); err != nil {
return
}
if err = service.DefaultSettings.UpdateCurrent(ctx); err != nil {
return
}
if err = oidcAutoDiscovery(ctx, log); err != nil {
return
}
if provisioned {
return partialImportSettings(ctx, service.DefaultSettings, readers...)
}
return errors.Wrap(
importer.Import(ctx, readers...),
"could not provision configuration for system service",
)
return nil
}
// Provision ONLY when there are no rules for role admins
func isProvisioned(ctx context.Context) (bool, error) {
return len(service.DefaultPermissions.FindRulesByRoleID(permissions.AdminsRoleID)) > 0, nil
// provision ONLY when there are no rules for role admins
func notProvisioned(ctx context.Context) (bool, error) {
return len(service.DefaultPermissions.FindRulesByRoleID(permissions.AdminsRoleID)) == 0, nil
}
func makeDefaultApplications(ctx context.Context, log *zap.Logger) error {
@@ -151,7 +171,7 @@ func makeDefaultApplications(ctx context.Context, log *zap.Logger) error {
}
// Partial import of settings from provision files
func partialImportSettings(ctx context.Context, ss settings.Service, ff ...io.Reader) (err error) {
func partialImportSettings(ctx context.Context, ss settingsService, ff ...io.Reader) (err error) {
var (
// decoded content from YAML files
aux interface{}
@@ -163,10 +183,10 @@ func partialImportSettings(ctx context.Context, ss settings.Service, ff ...io.Re
imp = importer.NewImporter(nil, si, nil)
// current value
current settings.ValueSet
current types.SettingValueSet
// unexisting values
unex settings.ValueSet
unex types.SettingValueSet
)
for _, f := range ff {
provision/system/src/0100_settings.yaml
+17
View File
@@ -64,3 +64,20 @@ settings:
<p>Follow <a href="{{ .URL }}" style="color:#568ba2;">this link</a> and reset your password.</p>
<p>You will be logged-in after successful reset.</p>
{{.EmailFooterEn}}
messaging.ui.emoji.enabled: true
messaging.ui.browser-notifications.enabled: true
messaging.ui.browser-notifications.header: ${user} in ${channel}
messaging.ui.browser-notifications.message-trim: 200
messaging.message.attachments.enabled: true
messaging.message.attachments.max-size: 10
messaging.message.attachments.mimetypes: []
messaging.message.attachments.source.gallery.enabled: true
messaging.message.attachments.source.camera.enabled: true
compose.ui.namespace-switcher.enabled: false
compose.record.attachments.max-size: 10
compose.record.attachments.mimetypes: []
compose.page.attachments.max-size: 10
compose.page.attachments.mimetypes: []
provision/system/static.go
+1 -1
View File
File diff suppressed because one or more lines are too long