DeforaNetworks/corteza
3
0
Fork 0
Code Activity

Cleanup and simplify provisioning

Browse Source
This commit is contained in:
Denis Arh
2020-11-30 13:13:46 +01:00
parent c299a64086
commit 1d47cee2f2
38 changed files with 248 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/boot_levels.go
+3 -22
View File
@@ -3,7 +3,6 @@ package app
import (
"context"
"crypto/tls"
"errors"
"fmt"
cmpService "github.com/cortezaproject/corteza-server/compose/service"
cmpEvent "github.com/cortezaproject/corteza-server/compose/service/event"
@@ -19,12 +18,10 @@ import (
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/pkg/mail"
"github.com/cortezaproject/corteza-server/pkg/monitor"
"github.com/cortezaproject/corteza-server/pkg/provision"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/pkg/scheduler"
"github.com/cortezaproject/corteza-server/pkg/sentry"
"github.com/cortezaproject/corteza-server/provision/compose"
"github.com/cortezaproject/corteza-server/provision/messaging"
"github.com/cortezaproject/corteza-server/provision/system"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/auth/external"
sysService "github.com/cortezaproject/corteza-server/system/service"
@@ -256,28 +253,12 @@ func (app *CortezaApp) Provision(ctx context.Context) (err error) {
if !app.Opt.Provision.Always {
app.Log.Debug("provisioning skipped (PROVISION_ALWAYS=false)")
} else {
ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_APP_Provision)
defer sentry.Recover()
ctx = actionlog.RequestOriginToContext(ctx, actionlog.RequestOrigin_APP_Provision)
ctx = auth.SetSuperUserContext(ctx)
if err = system.Provision(ctx, app.Log, app.Store); err != nil {
return fmt.Errorf("could not provision system: %w", err)
}
if err = compose.Provision(ctx, app.Log, app.Store); err != nil {
return fmt.Errorf("could not provision compose: %w", err)
}
if err = messaging.Provision(ctx, app.Log, app.Store); err != nil {
return fmt.Errorf("could not provision messaging: %w", err)
}
for errors.Unwrap(err) != nil {
err = errors.Unwrap(err)
}
if err != nil {
if err = provision.Run(ctx, app.Log, app.Store); err != nil {
return err
}
provision/system/auth_autodiscovery.go → pkg/provision/auth.go
+14 -9
View File
@@ -1,19 +1,24 @@
package system
package provision
import (
"context"
"fmt"
"github.com/cortezaproject/corteza-server/system/types"
"net/url"
"os"
"strings"
"github.com/spf13/cast"
"go.uber.org/zap"
"github.com/cortezaproject/corteza-server/pkg/logger"
"github.com/cortezaproject/corteza-server/pkg/rand"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
"github.com/spf13/cast"
"go.uber.org/zap"
"net/url"
"os"
"strings"
)
type (
settingsService interface {
FindByPrefix(context.Context, ...string) (types.SettingValueSet, error)
BulkSet(context.Context, types.SettingValueSet) error
}
)
var (
pkg/provision/config.go
+45
View File
@@ -0,0 +1,45 @@
package provision
import (
"context"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/directory"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
es "github.com/cortezaproject/corteza-server/pkg/envoy/store"
"github.com/cortezaproject/corteza-server/pkg/envoy/yaml"
"github.com/cortezaproject/corteza-server/store"
"go.uber.org/zap"
"path/filepath"
)
func importConfig(ctx context.Context, log *zap.Logger, s store.Storer) error {
var (
yd = yaml.Decoder()
nn = make([]resource.Interface, 0, 200)
se = es.NewStoreEncoder(s, &es.EncoderConfig{OnExisting: es.Skip})
bld = envoy.NewBuilder(se)
pp, err = filepath.Glob("provision/*")
)
if err != nil {
return err
}
for _, path := range pp {
log.Info("provisioning from path", zap.String("path", path))
if mm, err := directory.Decode(ctx, path, yd); err != nil {
return err
} else {
nn = append(nn, mm...)
}
}
if g, err := bld.Build(ctx, nn...); err != nil {
return err
} else if err = envoy.Encode(ctx, g, se); err != nil {
return err
}
return nil
}
provision/system/oidc.go → pkg/provision/oidc.go
+1 -1
View File
@@ -1,4 +1,4 @@
package system
package provision
import (
"context"
pkg/provision/provision.go
+27
View File
@@ -0,0 +1,27 @@
package provision
import (
"context"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/service"
"go.uber.org/zap"
)
func Run(ctx context.Context, log *zap.Logger, s store.Storer) error {
ffn := []func() error{
func() error { return roles(ctx, s) },
func() error { return importConfig(ctx, log, s) },
func() error { return authSettingsAutoDiscovery(ctx, log, service.DefaultSettings) },
func() error { return authAddExternals(ctx, log) },
func() error { return service.DefaultSettings.UpdateCurrent(ctx) },
func() error { return oidcAutoDiscovery(ctx, log) },
}
for _, fn := range ffn {
if err := fn(); err != nil {
return err
}
}
return nil
}
pkg/provision/roles.go
+30
View File
@@ -0,0 +1,30 @@
package provision
import (
"context"
"time"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/types"
)
func roles(ctx context.Context, s store.Storer) error {
if set, _, err := store.SearchRoles(ctx, s, types.RoleFilter{}); err != nil {
return err
} else if len(set) > 0 {
return nil
}
rr := types.RoleSet{
&types.Role{ID: rbac.AdminsRoleID, Name: "Administrators", Handle: "admins"},
&types.Role{ID: rbac.EveryoneRoleID, Name: "Everyone", Handle: "everyone"},
}
err := rr.Walk(func(r *types.Role) error {
r.CreatedAt = time.Now()
return store.CreateRole(ctx, s, r)
})
return err
}
provision/compose/src/system/0000_access_control.yaml → provision/000_base/compose_access_control.yaml
View File
provision/000_base/roles.yaml
+3
View File
@@ -0,0 +1,3 @@
roles:
- everyone
-
provision/system/src/0000_access_control.yaml → provision/000_base/system_access_control.yaml
View File
provision/001_settings/auth.yaml
+18
View File
@@ -0,0 +1,18 @@
settings:
auth.mail.email-confirmation.subject.en: Confirm your email address
auth.mail.email-confirmation.body.en: |-
{{.EmailHeaderEn}}
<h2 style="color: #568ba2;text-align: center;">Confirm your email address</h2>
<p>Hello,</p>
<p>Follow <a href="{{ .URL }}" style="color:#568ba2;">this link</a> to confirm your email address.</p>
<p>You will be logged-in after successful confirmation.</p>
{{.EmailFooterEn}}
auth.mail.password-reset.subject.en: Reset your password
auth.mail.password-reset.body.en: |-
{{.EmailHeaderEn}}
<h2 style="color: #568ba2;text-align: center;">Reset your password</h2>
<p>Hello,</p>
<p>Follow <a href="{{ .URL }}" style="color:#568ba2;">this link</a> and reset your password.</p>
<p>You will be logged-in after successful reset.</p>
{{.EmailFooterEn}}
provision/001_settings/compose.yaml
+7
View File
@@ -0,0 +1,7 @@
settings:
compose.ui.namespace-switcher.enabled: false
compose.record.attachments.max-size: 10
compose.record.attachments.mimetypes: []
compose.page.attachments.max-size: 10
compose.page.attachments.mimetypes: []
provision/001_settings/general.yaml
+45
View File
File diff suppressed because one or more lines are too long
provision/001_settings/privacy.yaml
+3
View File
@@ -0,0 +1,3 @@
settings:
privacy.mask.email: true
privacy.mask.name: true
provision/system/src/0100_settings.yaml → provision/001_settings/system.yaml
-35
View File
@@ -46,38 +46,3 @@ settings:
</tbody>
</table>
</div>
auth.mail.email-confirmation.subject.en: Confirm your email address
auth.mail.email-confirmation.body.en: |-
{{.EmailHeaderEn}}
<h2 style="color: #568ba2;text-align: center;">Confirm your email address</h2>
<p>Hello,</p>
<p>Follow <a href="{{ .URL }}" style="color:#568ba2;">this link</a> to confirm your email address.</p>
<p>You will be logged-in after successful confirmation.</p>
{{.EmailFooterEn}}
auth.mail.password-reset.subject.en: Reset your password
auth.mail.password-reset.body.en: |-
{{.EmailHeaderEn}}
<h2 style="color: #568ba2;text-align: center;">Reset your password</h2>
<p>Hello,</p>
<p>Follow <a href="{{ .URL }}" style="color:#568ba2;">this link</a> and reset your password.</p>
<p>You will be logged-in after successful reset.</p>
{{.EmailFooterEn}}
messaging.ui.emoji.enabled: true
messaging.ui.browser-notifications.enabled: true
messaging.ui.browser-notifications.header: ${user} in ${channel}
messaging.ui.browser-notifications.message-trim: 200
messaging.message.attachments.enabled: true
messaging.message.attachments.max-size: 10
messaging.message.attachments.mimetypes: []
messaging.message.attachments.source.gallery.enabled: true
messaging.message.attachments.source.camera.enabled: true
compose.ui.namespace-switcher.enabled: false
compose.record.attachments.max-size: 10
compose.record.attachments.mimetypes: []
compose.page.attachments.max-size: 10
compose.page.attachments.mimetypes: []
provision/system/src/0200_applications.yaml → provision/101_applications/0200_applications.yaml
View File
provision/compose/src/crm/1000_namespace.yaml → provision/700_crm/1000_namespace.yaml
View File
provision/compose/src/crm/1100_modules.yaml → provision/700_crm/1100_modules.yaml
View File
provision/compose/src/crm/1200_charts.yaml → provision/700_crm/1200_charts.yaml
View File
provision/compose/src/crm/1400_pages.yaml → provision/700_crm/1400_pages.yaml
View File
provision/compose/src/crm/1500_record_settings.yaml → provision/700_crm/1500_record_settings.yaml
View File
provision/compose/src/service-solution/1000_namespace.yaml → provision/701_service-solution/1000_namespace.yaml
View File
provision/compose/src/service-solution/1100_modules.yaml → provision/701_service-solution/1100_modules.yaml
View File
provision/compose/src/service-solution/1200_charts.yaml → provision/701_service-solution/1200_charts.yaml
View File
provision/compose/src/service-solution/1400_pages.yaml → provision/701_service-solution/1400_pages.yaml
View File
provision/compose/src/service-solution/1500_record_settings.yaml → provision/701_service-solution/1500_record_settings.yaml
View File
provision/messaging/src/0000_access_control.yaml → provision/999_messaging/messaging_access_control.yaml
View File
provision/messaging/src/1000_channels.yaml → provision/999_messaging/messaging_channels.yaml
View File
provision/999_messaging/messaging_settings.yaml
+10
View File
@@ -0,0 +1,10 @@
settings:
messaging.ui.emoji.enabled: true
messaging.ui.browser-notifications.enabled: true
messaging.ui.browser-notifications.header: ${user} in ${channel}
messaging.ui.browser-notifications.message-trim: 200
messaging.message.attachments.enabled: true
messaging.message.attachments.max-size: 10
messaging.message.attachments.mimetypes: []
messaging.message.attachments.source.gallery.enabled: true
messaging.message.attachments.source.camera.enabled: true
provision/Makefile
-14
View File
@@ -1,14 +0,0 @@
.PHONY: clean all
include ../Makefile.inc
PARTS ?= compose messaging system
OUTPUTS ?= $(addsuffix /static.go, $(PARTS))
all: $(OUTPUTS)
%/static.go: $(STATIK)
$(STATIK) -p $(@D) -m -Z -f -src=$(@D)/src
clean:
rm -f $(OUTPUTS)
provision/README.adoc
+28
View File
@@ -0,0 +1,28 @@
= Provision config files
By default, Corteza provisions itself with all config files found in this directory.
Provisioning process can be configured via `PROVISION_CONFIG_PATHS`.
Extensions (`8xx-xxxxx`) are installed from https://github.com/cortezaproject/corteza-ext by release procedure.
Manual installation of extensions is also possible by running `./update.sh` shell script in this directory
== Naming/numeration rules
0xx::
Base resources, configuration, access control rules
1xx::
Additional resources
2xx..6xx::
Reserved
7xx::
Reserved for extensions
8xx::
Reserved for custom/user configuration
9xx::
Temporary, deprecated, obsolete
provision/compose/provision.go
-53
View File
@@ -1,53 +0,0 @@
package compose
import (
"context"
"github.com/cortezaproject/corteza-server/provision/util"
"github.com/cortezaproject/corteza-server/store"
"go.uber.org/zap"
)
func hasNamespaces(ctx context.Context, s store.Storer, slug string) (bool, error) {
ns, err := store.LookupComposeNamespaceBySlug(ctx, s, slug)
if err == store.ErrNotFound {
return false, nil
} else if err != nil {
return false, err
}
return ns != nil, nil
}
func Provision(ctx context.Context, log *zap.Logger, s store.Storer) error {
log.Info("provisioning compose: system")
if err := util.EncodeStatik(ctx, s, Asset, "/system"); err != nil {
return err
}
log.Info("provisioned compose: system")
if has, err := hasNamespaces(ctx, s, "crm"); !has && err == nil {
log.Info("provisioning compose: crm")
if err := util.EncodeStatik(ctx, s, Asset, "/crm"); err != nil {
return err
}
log.Info("provisioned compose: crm")
} else if err != nil {
return err
} else {
log.Info("[skip] provisioning compose: crm")
}
if has, err := hasNamespaces(ctx, s, "service-solution"); !has && err == nil {
log.Info("provisioning compose: service-solution")
if err := util.EncodeStatik(ctx, s, Asset, "/service-solution"); err != nil {
return err
}
log.Info("provisioned compose: service-solution")
} else if err != nil {
return err
} else {
log.Info("[skip] provisioning compose: service-solution")
}
return nil
}
provision/compose/static.go
-6
View File
File diff suppressed because one or more lines are too long
provision/messaging/provision.go
-35
View File
@@ -1,35 +0,0 @@
package messaging
import (
"context"
"github.com/cortezaproject/corteza-server/messaging/types"
"github.com/cortezaproject/corteza-server/provision/util"
"github.com/cortezaproject/corteza-server/store"
"go.uber.org/zap"
)
// provision only where there are no channels
func hasChannels(ctx context.Context, s store.Storer) (bool, error) {
if set, _, err := store.SearchMessagingChannels(ctx, s, types.ChannelFilter{IncludeDeleted: true}); err != nil {
return false, err
} else {
return len(set) > 0, nil
}
}
func Provision(ctx context.Context, log *zap.Logger, s store.Storer) error {
log.Info("provisioning messaging")
if channelsExist, err := hasChannels(ctx, s); err != nil {
return err
} else if !channelsExist {
// Provision from YAML files
// - access control
// - channels
if err = util.EncodeStatik(ctx, s, Asset, "/"); err != nil {
return err
}
}
log.Info("messaging provisioned")
return nil
}
provision/messaging/static.go
-6
View File
@@ -1,6 +0,0 @@
// Code generated by statik. DO NOT EDIT.
// Package contains static assets.
package messaging
var Asset = "PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x18\x00 \x000000_access_control.yamlUT\x05\x00\x01\x80Cm8allow:\n everyone:\n messaging:\n - access\n\n admins:\n messaging:\n - access\n - grant\n - settings.read\n - settings.manage\n - channel.public.create\n - channel.private.create\n - channel.group.create\n\n messaging:channel:\n - update\n - leave\n - read\n - join\n - delete\n - undelete\n - archive\n - unarchive\n - members.manage\n - attachments.manage\n - message.attach\n - message.update.all\n - message.update.own\n - message.delete.all\n - message.delete.own\n - message.embed\n - message.send\n - message.reply\n - message.react\n\nPK\x07\x08\xae\x83\x83d\x8e\x02\x00\x00\x8e\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x12\x00 \x001000_channels.yamlUT\x05\x00\x01\x80Cm8channels:\n - name: General\n type: public\n - name: Random\n type: public\nPK\x07\x08\xe8\x83F\xf8O\x00\x00\x00O\x00\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x00\x00!(\xae\x83\x83d\x8e\x02\x00\x00\x8e\x02\x00\x00\x18\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x000000_access_control.yamlUT\x05\x00\x01\x80Cm8PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x00\x00!(\xe8\x83F\xf8O\x00\x00\x00O\x00\x00\x00\x12\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xdd\x02\x00\x001000_channels.yamlUT\x05\x00\x01\x80Cm8PK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00\x98\x00\x00\x00u\x03\x00\x00\x00\x00"
provision/system/provision.go
-76
View File
@@ -1,76 +0,0 @@
package system
import (
"context"
"time"
"github.com/cortezaproject/corteza-server/pkg/rbac"
"github.com/cortezaproject/corteza-server/provision/util"
"github.com/cortezaproject/corteza-server/store"
"github.com/cortezaproject/corteza-server/system/service"
"github.com/cortezaproject/corteza-server/system/types"
"go.uber.org/zap"
)
type (
settingsService interface {
FindByPrefix(context.Context, ...string) (types.SettingValueSet, error)
BulkSet(context.Context, types.SettingValueSet) error
}
)
func provisionRoles(ctx context.Context, s store.Storer) error {
if set, _, err := store.SearchRoles(ctx, s, types.RoleFilter{}); err != nil {
return err
} else if len(set) > 0 {
return nil
}
rr := types.RoleSet{
&types.Role{ID: rbac.AdminsRoleID, Name: "Administrators", Handle: "admins"},
&types.Role{ID: rbac.EveryoneRoleID, Name: "Everyone", Handle: "everyone"},
}
err := rr.Walk(func(r *types.Role) error {
r.CreatedAt = time.Now()
return store.CreateRole(ctx, s, r)
})
return err
}
func Provision(ctx context.Context, log *zap.Logger, s store.Storer) (err error) {
log.Info("provisioning system")
err = provisionRoles(ctx, s)
if err != nil {
return err
}
// Provision from YAML files
// - access control
// - settings
// - applications
if err = util.EncodeStatik(ctx, s, Asset, "/"); err != nil {
return err
}
// These ones need some extra things, so we'll leave them there
if err = authSettingsAutoDiscovery(ctx, log, service.DefaultSettings); err != nil {
return
}
if err = authAddExternals(ctx, log); err != nil {
return
}
if err = service.DefaultSettings.UpdateCurrent(ctx); err != nil {
return
}
if err = oidcAutoDiscovery(ctx, log); err != nil {
return
}
log.Info("provisioned system")
return nil
}
provision/system/static.go
-6
View File
File diff suppressed because one or more lines are too long
provision/update.sh
+14 -14
View File
@@ -4,7 +4,7 @@ set -eu
cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
BRANCH=${BRANCH:-"develop"}
BRANCH=${BRANCH:-"2020.12.x"}
ZIP="${BRANCH}.zip"
URL=${URL:-"https://github.com/cortezaproject/corteza-ext/archive/${ZIP}"}
DIR="corteza-ext-${BRANCH}"
@@ -19,9 +19,9 @@ function download {
}
function copyExtConfig {
echo -e "\033[32mCopying ${2} ${1}\033[39m ... "
mkdir -p "./compose/src/${1}"
cp "${DIR}/${1}/config/${2}.yaml" "./compose/src/${1}/${2}.yaml"
echo -e "\033[32mCopying ${3} ${2}\033[39m ... "
mkdir -p "./${1}_${2}"
cp -f "${DIR}/${2}/config/${3}.yaml" "./${1}_${2}/${3}.yaml"
}
function cleanup {
@@ -32,17 +32,17 @@ function cleanup {
download
copyExtConfig crm 1000_namespace
copyExtConfig crm 1100_modules
copyExtConfig crm 1200_charts
copyExtConfig crm 1400_pages
copyExtConfig crm 1500_record_settings
copyExtConfig 700 crm 1000_namespace
copyExtConfig 700 crm 1100_modules
copyExtConfig 700 crm 1200_charts
copyExtConfig 700 crm 1400_pages
copyExtConfig 700 crm 1500_record_settings
copyExtConfig service-solution 1000_namespace
copyExtConfig service-solution 1100_modules
copyExtConfig service-solution 1200_charts
copyExtConfig service-solution 1400_pages
copyExtConfig service-solution 1500_record_settings
copyExtConfig 701 service-solution 1000_namespace
copyExtConfig 701 service-solution 1100_modules
copyExtConfig 701 service-solution 1200_charts
copyExtConfig 701 service-solution 1400_pages
copyExtConfig 701 service-solution 1500_record_settings
cleanup
provision/util/decode_statik.go
-95
View File
@@ -1,95 +0,0 @@
package util
import (
"context"
"fmt"
"io"
"net/http"
"os"
"path"
"github.com/cortezaproject/corteza-server/pkg/envoy"
"github.com/cortezaproject/corteza-server/pkg/envoy/resource"
es "github.com/cortezaproject/corteza-server/pkg/envoy/store"
"github.com/cortezaproject/corteza-server/pkg/envoy/yaml"
"github.com/cortezaproject/corteza-server/store"
"github.com/goware/statik/fs"
)
type (
decoder interface {
CanDecodeExt(string) bool
CanDecodeFile(io.Reader) bool
Decode(context.Context, io.Reader, *envoy.DecoderOpts) ([]resource.Interface, error)
}
)
func StatikResources(ctx context.Context, d decoder, sfs http.FileSystem, root string) ([]resource.Interface, error) {
nn := make(resource.InterfaceSet, 0, 100)
err := fs.Walk(sfs, root, func(p string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
f, err := sfs.Open(p)
if err != nil {
return err
}
defer f.Close()
if !d.CanDecodeFile(f) && !d.CanDecodeExt(info.Name()) {
return nil
}
if d == nil {
// no decoder found
return nil
}
_, err = f.Seek(0, 0)
if err != nil {
return err
}
dir, fn := path.Split(p)
do := &envoy.DecoderOpts{
Name: fn,
Path: dir,
}
if dnn, err := d.Decode(ctx, f, do); err != nil {
return fmt.Errorf("failed to decode %s: %w", info.Name(), err)
} else {
nn = append(nn, dnn...)
}
return nil
})
return nn, err
}
func EncodeStatik(ctx context.Context, s store.Storer, rfs string, root string) error {
sfs, err := fs.New(rfs)
if err != nil {
return err
}
yd := yaml.Decoder()
se := es.NewStoreEncoder(s, &es.EncoderConfig{
OnExisting: es.MergeLeft,
})
nn, err := StatikResources(ctx, yd, sfs, root)
if err != nil {
return err
}
bld := envoy.NewBuilder(se)
g, err := bld.Build(ctx, nn...)
if err != nil {
return err
}
return envoy.Encode(ctx, g, se)
}